This is an automated email from the ASF dual-hosted git repository. vishesh92 pushed a commit to branch add-simulator-e2e-environment in repository https://gitbox.apache.org/repos/asf/cloudstack-kubernetes-provider.git
commit 9e0a52bc5d6b9a3c0a94c9498f8f84b443301a7d Author: vishesh92 <[email protected]> AuthorDate: Mon Aug 31 13:53:02 2026 +0530 Add a simulated dev/test environment and simulator-based e2e CI The repository had no way to exercise the CCM end to end. The only "run against real CloudStack" hook was configFromEnv() in cloudstack_test.go, which skips unless CS_API_URL and friends are set, and nothing set them. As a result EnsureLoadBalancer, UpdateLoadBalancer and EnsureLoadBalancerDeleted -- the three functions holding nearly all of the load balancer branching -- had no test coverage at all, and the README pointed at a Docker Hub image (cloudstack/simulator) that no longer exists. Add hack/e2e, which brings up a CloudStack simulator, deploys its advanced zone, mints admin API keys, creates a kind cluster and deploys CloudStack VMs matching its nodes, then runs the CCM against both. docs/development.md documents the same steps by hand so the environment is understandable rather than magic. Add a Go e2e suite under test/e2e covering load balancer lifecycle, node initialization, service annotations and the VPC/network ACL path. It is behind the e2e build tag, so it stays out of `make test` and `go build ./...`, and it needs no new module dependencies. Run all of it in CI as a matrix of the latest two Kubernetes versions against the latest two CloudStack releases. The CloudStack axis is not only version coverage: 4.22 and later update a load balancer rule's CIDR list in place while earlier releases delete and recreate the rule, so both branches are exercised. Cells run in parallel and share a single image build, so the workflow costs about as much wall-clock as one run. Fix two project-scoping bugs the new suite uncovered. updateNetworkACL fetched the network and ACL list without the project, so every LoadBalancer service on a VPC owned by a project failed with "error fetching Network with ID" and never got an ingress address. getNetworkIDFromIPAddress had the same omission, breaking load balancer deletion for projects. Also add the local cloud-config, cmk-config and kube-config files to .gitignore. They hold live credentials and were previously untracked but not ignored. Fixes #4 Co-Authored-By: Claude Opus 5 <[email protected]> --- .github/workflows/e2e-simulator.yml | 180 +++++++++++++++++ .gitignore | 6 + .golangci.yml | 3 + Makefile | 18 +- README.md | 64 +++--- cloudstack_loadbalancer.go | 8 +- cloudstack_loadbalancer_test.go | 32 +-- docs/development.md | 381 ++++++++++++++++++++++++++++++++++++ hack/e2e/10-simulator-up.sh | 111 +++++++++++ hack/e2e/20-kind-up.sh | 52 +++++ hack/e2e/30-topology-isolated.sh | 87 ++++++++ hack/e2e/40-ccm-deploy.sh | 93 +++++++++ hack/e2e/50-topology-vpc.sh | 123 ++++++++++++ hack/e2e/90-collect-artifacts.sh | 64 ++++++ hack/e2e/99-down.sh | 38 ++++ hack/e2e/env.sh | 65 ++++++ hack/e2e/kind-config.yaml | 62 ++++++ hack/e2e/lib/cs.sh | 89 +++++++++ hack/e2e/lib/log.sh | 56 ++++++ hack/e2e/up.sh | 38 ++++ test/e2e/annotations_test.go | 176 +++++++++++++++++ test/e2e/framework.go | 381 ++++++++++++++++++++++++++++++++++++ test/e2e/loadbalancer_test.go | 213 ++++++++++++++++++++ test/e2e/node_test.go | 137 +++++++++++++ test/e2e/vpc_test.go | 132 +++++++++++++ 25 files changed, 2549 insertions(+), 60 deletions(-) diff --git a/.github/workflows/e2e-simulator.yml b/.github/workflows/e2e-simulator.yml new file mode 100644 index 00000000..bf63ff2a --- /dev/null +++ b/.github/workflows/e2e-simulator.yml @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: E2E (CloudStack simulator) + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # Build the CCM image once and share it with every matrix cell. The + # distroless image is small enough that passing it as an artifact is much + # cheaper than four redundant builds. + build: + if: github.repository == 'apache/cloudstack-kubernetes-provider' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + + - uses: docker/setup-buildx-action@v3 + + - name: Build CCM image + uses: docker/build-push-action@v6 + with: + context: . + load: true + platforms: linux/amd64 + tags: apache/cloudstack-kubernetes-provider:e2e + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Export image + run: docker save apache/cloudstack-kubernetes-provider:e2e | zstd -T0 -o ccm-image.tar.zst + + - uses: actions/upload-artifact@v4 + with: + name: ccm-image + path: ccm-image.tar.zst + retention-days: 1 + + e2e: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # Latest two Kubernetes minors and latest two CloudStack releases. + # The CloudStack axis is not just version coverage: >= 4.22 updates a + # load balancer rule's CIDR list in place, while older versions delete + # and recreate the rule, so both branches get exercised. + k8s: ['v1.37.0', 'v1.36.4'] + acs: ['4.22.1.0', '4.21.0.0'] + env: + KIND_NODE_IMAGE: kindest/node:${{ matrix.k8s }} + SIM_TAG: ${{ matrix.acs }} + CCM_IMAGE: apache/cloudstack-kubernetes-provider:e2e + # The secrets context is not available in a step-level `if`, so resolve + # it once here. Forks without the secret still run, just unauthenticated. + HAS_DOCKERHUB_CREDS: ${{ secrets.DOCKERHUB_TOKEN != '' }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Free disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/boost "${AGENT_TOOLSDIRECTORY:-}" + docker system prune -af + df -h / + + # Authenticated pulls avoid Docker Hub's anonymous rate limit, which + # four parallel 2 GB simulator pulls would otherwise run into. + - name: Log in to Docker Hub + if: env.HAS_DOCKERHUB_CREDS == 'true' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Pull simulator image + run: | + for i in 1 2 3; do + docker pull "apache/cloudstack-simulator:${SIM_TAG}" && exit 0 + echo "pull attempt $i failed, retrying" + sleep 30 + done + exit 1 + + - uses: actions/download-artifact@v4 + with: + name: ccm-image + + - name: Load CCM image + run: zstd -dc ccm-image.tar.zst | docker load + + - uses: helm/kind-action@v1 + with: + install_only: true + version: v0.32.0 + + - name: Start simulator and deploy zone + run: hack/e2e/10-simulator-up.sh + + - name: Create kind cluster + run: hack/e2e/20-kind-up.sh + + - name: Create isolated network topology + run: hack/e2e/30-topology-isolated.sh + + - name: Deploy the cloud controller manager + run: hack/e2e/40-ccm-deploy.sh + + - name: Unit and acceptance tests against the live simulator + run: | + . hack/e2e/_out/keys.env + CS_API_URL=http://localhost:8080/client/api make test + + - name: E2E phase 1 (load balancer, nodes, annotations) + run: | + . hack/e2e/_out/keys.env + KUBECONFIG="${PWD}/hack/e2e/_out/kubeconfig" \ + CS_API_URL=http://localhost:8080/client/api \ + go test -tags e2e -v -timeout 30m ./test/e2e/... -run 'TestLB|TestNode|TestAnnot' + + - name: Create VPC topology + run: hack/e2e/50-topology-vpc.sh + + - name: E2E phase 2 (VPC / network ACL) + run: | + . hack/e2e/_out/keys.env + . hack/e2e/_out/ids.env + KUBECONFIG="${PWD}/hack/e2e/_out/kubeconfig" \ + CS_API_URL=http://localhost:8080/client/api \ + CS_PROJECT_ID="${E2E_PROJECT_ID}" \ + E2E_ACL_ID="${E2E_ACL_ID}" E2E_VPC_ID="${E2E_VPC_ID}" \ + go test -tags e2e -v -timeout 30m ./test/e2e/... -run 'TestVPC' + + - name: Collect artifacts + if: always() + run: hack/e2e/90-collect-artifacts.sh + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: e2e-artifacts-${{ matrix.k8s }}-${{ matrix.acs }} + path: hack/e2e/_out/artifacts + retention-days: 7 + + - name: Tear down + if: always() + run: hack/e2e/99-down.sh diff --git a/.gitignore b/.gitignore index 79e12562..73c4ecea 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,12 @@ go.work.sum # env file .env +# Local dev/test credentials and generated harness state — never commit these +/cloud-config +/cmk-config +/kube-config +/hack/e2e/_out/ + # Editor/IDE .idea/ .vscode/ \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml index 379c873f..84ae959e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -19,6 +19,9 @@ version: "2" run: modules-download-mode: readonly issues-exit-code: 1 + # Lint the build-tagged e2e suite too; without this, goheader/gosec silently skip it. + build-tags: + - e2e linters: enable: - goheader diff --git a/Makefile b/Makefile index bbb03f0d..2d336a8b 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ export GO111MODULE=on CMD_SRC=\ cmd/cloudstack-ccm/main.go -.PHONY: all clean docker +.PHONY: all clean docker e2e-up e2e-down test-e2e all: cloudstack-ccm @@ -53,6 +53,22 @@ ifneq (${GIT_IS_TAG},NOT_A_TAG) docker tag apache/cloudstack-kubernetes-provider:${GIT_COMMIT_SHORT} apache/cloudstack-kubernetes-provider:${GIT_TAG} endif +# Simulator-based e2e environment; see docs/development.md +e2e-up: + hack/e2e/up.sh + +e2e-down: + hack/e2e/99-down.sh + +# go test runs with the package directory as its working directory, so +# KUBECONFIG must be absolute. +test-e2e: + @test -f hack/e2e/_out/keys.env || (echo "environment not up; run 'make e2e-up' first" && exit 1) + . hack/e2e/_out/keys.env && \ + KUBECONFIG=${CURDIR}/hack/e2e/_out/kubeconfig \ + CS_API_URL=http://localhost:8080/client/api \ + go test -tags e2e -v -timeout 30m ./test/e2e/... -run 'TestLB|TestNode|TestAnnot' + lint: gofmt @(echo "Running golangci-lint...") golangci-lint run diff --git a/README.md b/README.md index fc4922b7..53f50a82 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,11 @@ explicitly set `region` in that case. The access token needs to be able to fetch VM information and deploy load balancers in the project or domain where the nodes reside. +The account must also be allowed to call `listManagementServersMetrics`, which the controller uses +on startup to determine the management server version. This is a root admin API and is **not** +included in the default `User` role; without it the controller exits immediately with +`no management servers found`. + To create the secret, use the following command: ```bash kubectl -n kube-system create secret generic cloudstack-secret --from-file=cloud-config @@ -423,9 +428,15 @@ make docker ### Testing -You need a local instance of the CloudStack Management Server or a 'real' one to connect to. +Unit tests need nothing but Go: + +```bash +make test +``` + +For anything beyond that you need a CloudStack Management Server to talk to. The CCM supports the same cloud-config configuration file format used by [the cs tool](https://github.com/exoscale/cs), -so you can simply point it to that. +so you can simply point it at one you already have: ```bash ./cloudstack-ccm --cloud-provider external-cloudstack --cloud-config ./cloud-config --kubeconfig ~/.kube/config @@ -434,45 +445,20 @@ so you can simply point it to that. Point `--kubeconfig` at a kubeconfig for your Kubernetes development cluster, and `--cloud-config` at a `cloud-config` for the CloudStack installation you want to talk to. -If you don't have a 'real' CloudStack installation, you can also launch a local [simulator instance](https://hub.docker.com/r/cloudstack/simulator) instead. This is very useful for dry-run testing. - -### Debugging - -You can use the VSCode extension [Go](https://marketplace.visualstudio.com/items?itemName=golang.go) to debug the CCM. -Add the following configuration to the `.vscode/launch.json` file to launch the CCM and debug it. - -```json -{ - "version": "0.2.0", - "configurations": [ - { - "name": "Launch CloudStack CCM", - "type": "go", - "request": "launch", - "mode": "auto", - "program": "${workspaceFolder}/cmd/cloudstack-ccm", - "env": {}, - "args": [ - "--cloud-provider=external-cloudstack", - "--cloud-config=${workspaceFolder}/cloud-config", - "--kubeconfig=${env:HOME}/.kube/config", - "--leader-elect=false", - "--v=4" - ], - "showLog": true, - "trace": "verbose" - }, - { - "name": "Attach to Process", - "type": "go", - "request": "attach", - "mode": "local", - "processId": 0 - } - ] -} +If you don't have a 'real' CloudStack installation, you don't need one. The repository ships a +fully simulated environment — a kind cluster, the +[CloudStack simulator](https://hub.docker.com/r/apache/cloudstack-simulator) and the CCM, all in +containers: + +```bash +make e2e-up # bring the environment up +make test-e2e # run the end-to-end suite against it +make e2e-down # tear it down ``` +See [docs/development.md](docs/development.md) for the full walkthrough, how to run the CCM as a +host process under a debugger, the VPC scenario, and troubleshooting. + ## Copyright Copyright 2019 The Apache Software Foundation diff --git a/cloudstack_loadbalancer.go b/cloudstack_loadbalancer.go index ffbdd7cd..0e77e4ab 100644 --- a/cloudstack_loadbalancer.go +++ b/cloudstack_loadbalancer.go @@ -471,7 +471,7 @@ func (cs *CSCloud) getLoadBalancer(service *corev1.Service) (*loadBalancer, erro // Get network ID from Public IP Address func (cs *CSCloud) getNetworkIDFromIPAddress(publicIpId string) (string, error) { - ip, count, err := cs.client.Address.GetPublicIpAddressByID(publicIpId) + ip, count, err := cs.client.Address.GetPublicIpAddressByID(publicIpId, cloudstack.WithProject(cs.projectID)) if err != nil { klog.Errorf("Failed to fetch the public IP for id: %v", publicIpId) return "", err @@ -480,7 +480,7 @@ func (cs *CSCloud) getNetworkIDFromIPAddress(publicIpId string) (string, error) return "", err } if ip.Networkid != "" { - network, _, netErr := cs.client.Network.GetNetworkByID(ip.Associatednetworkid) + network, _, netErr := cs.client.Network.GetNetworkByID(ip.Associatednetworkid, cloudstack.WithProject(cs.projectID)) if netErr != nil { klog.Errorf("Failed to fetch the network for id: %v", ip.Associatednetworkid) return "", err @@ -981,12 +981,12 @@ func (lb *loadBalancer) updateFirewallRule(publicIpId string, publicPort int, pr } func (lb *loadBalancer) updateNetworkACL(publicPort int, protocol LoadBalancerProtocol, networkId string) (bool, error) { - network, _, err := lb.Network.GetNetworkByID(networkId) + network, _, err := lb.Network.GetNetworkByID(networkId, cloudstack.WithProject(lb.projectID)) if err != nil { return false, fmt.Errorf("error fetching Network with ID: %v, due to: %s", networkId, err) } - networkAclList, count, err := lb.NetworkACL.GetNetworkACLListByID(network.Aclid) + networkAclList, count, err := lb.NetworkACL.GetNetworkACLListByID(network.Aclid, cloudstack.WithProject(lb.projectID)) if err != nil { return false, fmt.Errorf("error fetching Network ACL List with ID: %v, due to: %s", network.Aclid, err) } diff --git a/cloudstack_loadbalancer_test.go b/cloudstack_loadbalancer_test.go index 4bbf38e7..878d3025 100644 --- a/cloudstack_loadbalancer_test.go +++ b/cloudstack_loadbalancer_test.go @@ -2829,8 +2829,8 @@ func TestUpdateNetworkACL(t *testing.T) { } gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(aclListResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(aclListResp, 1, nil), mockNetworkACL.EXPECT().NewListNetworkACLsParams().Return(listParams), mockNetworkACL.EXPECT().ListNetworkACLs(gomock.Any()).Return(listResp, nil), mockNetworkACL.EXPECT().NewCreateNetworkACLParams("tcp").Return(createParams), @@ -2884,8 +2884,8 @@ func TestUpdateNetworkACL(t *testing.T) { } gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(aclListResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(aclListResp, 1, nil), mockNetworkACL.EXPECT().NewListNetworkACLsParams().Return(listParams), mockNetworkACL.EXPECT().ListNetworkACLs(gomock.Any()).Return(listResp, nil), ) @@ -2924,8 +2924,8 @@ func TestUpdateNetworkACL(t *testing.T) { } gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(aclListResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(aclListResp, 1, nil), ) lb := &loadBalancer{ @@ -2951,7 +2951,7 @@ func TestUpdateNetworkACL(t *testing.T) { mockNetwork := cloudstack.NewMockNetworkServiceIface(ctrl) apiErr := fmt.Errorf("network API error") - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(nil, 1, apiErr) + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(nil, 1, apiErr) lb := &loadBalancer{ CloudStackClient: &cloudstack.CloudStackClient{ @@ -2983,8 +2983,8 @@ func TestUpdateNetworkACL(t *testing.T) { apiErr := fmt.Errorf("ACL list API error") gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(nil, 0, apiErr), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(nil, 0, apiErr), ) lb := &loadBalancer{ @@ -3024,8 +3024,8 @@ func TestUpdateNetworkACL(t *testing.T) { apiErr := fmt.Errorf("list ACL API error") gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(aclListResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(aclListResp, 1, nil), mockNetworkACL.EXPECT().NewListNetworkACLsParams().Return(listParams), mockNetworkACL.EXPECT().ListNetworkACLs(gomock.Any()).Return(nil, apiErr), ) @@ -3073,8 +3073,8 @@ func TestUpdateNetworkACL(t *testing.T) { apiErr := fmt.Errorf("create ACL API error") gomock.InOrder( - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), - mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456").Return(aclListResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), + mockNetworkACL.EXPECT().GetNetworkACLListByID("acl-456", gomock.Any()).Return(aclListResp, 1, nil), mockNetworkACL.EXPECT().NewListNetworkACLsParams().Return(listParams), mockNetworkACL.EXPECT().ListNetworkACLs(gomock.Any()).Return(listResp, nil), mockNetworkACL.EXPECT().NewCreateNetworkACLParams("tcp").Return(createParams), @@ -3407,8 +3407,8 @@ func TestGetNetworkIDFromIPAddress(t *testing.T) { } gomock.InOrder( - mockAddress.EXPECT().GetPublicIpAddressByID("ip-123").Return(ipResp, 1, nil), - mockNetwork.EXPECT().GetNetworkByID("net-123").Return(networkResp, 1, nil), + mockAddress.EXPECT().GetPublicIpAddressByID("ip-123", gomock.Any()).Return(ipResp, 1, nil), + mockNetwork.EXPECT().GetNetworkByID("net-123", gomock.Any()).Return(networkResp, 1, nil), ) cs := &CSCloud{ @@ -3434,7 +3434,7 @@ func TestGetNetworkIDFromIPAddress(t *testing.T) { mockAddress := cloudstack.NewMockAddressServiceIface(ctrl) apiErr := fmt.Errorf("IP not found") - mockAddress.EXPECT().GetPublicIpAddressByID("ip-123").Return(nil, 0, apiErr) + mockAddress.EXPECT().GetPublicIpAddressByID("ip-123", gomock.Any()).Return(nil, 0, apiErr) cs := &CSCloud{ client: &cloudstack.CloudStackClient{ diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..b4a99360 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,381 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +# Development + +This document describes how to run a fully simulated development and test +environment for the CloudStack Kubernetes Provider: a real Kubernetes API +server, a real CloudStack management server and the CCM itself, all in +containers on your workstation. Nothing is mocked — the CCM makes genuine +CloudStack API calls and the resulting load balancer rules, public IPs and +firewall/ACL rules are real database objects you can inspect. + +The pieces are: + +| Component | What provides it | +| --- | --- | +| Kubernetes API server + kubelets | a [kind](https://kind.sigs.k8s.io/) cluster | +| CloudStack management server | the [`apache/cloudstack-simulator`](https://hub.docker.com/r/apache/cloudstack-simulator) container | +| Cloud controller manager | this repository, either in-cluster or as a host process | + +## Prerequisites + +* Docker +* [kind](https://kind.sigs.k8s.io/) v0.30 or later +* `kubectl` +* Go 1.23 or later +* `jq` and `curl` +* About 12 GB of free disk and 8 GB of RAM + +> **linux/amd64 only.** `apache/cloudstack-simulator` is not published for +> arm64. On Apple Silicon you can run it under emulation with +> `--platform linux/amd64` (expect the simulator to take three to five times +> longer to start), point the environment at a simulator running on an x86 +> host, or build an arm64 image yourself from `tools/docker/` in the +> [apache/cloudstack](https://github.com/apache/cloudstack) repository. + +## Quickstart + +```bash +make e2e-up # simulator + zone + kind cluster + CloudStack VMs + CCM +make test-e2e # run the end-to-end suite +make e2e-down # tear everything down +``` + +`make e2e-up` takes about seven minutes once the simulator image is pulled — +roughly 90 seconds for the simulator to start, two and a half minutes to +deploy the zone, and the rest for the kind cluster, the VMs and the CCM. The +first run also has to pull a ~2 GB image. + +Once it is up, try the thing the CCM exists for: + +```bash +export KUBECONFIG=hack/e2e/_out/kubeconfig +kubectl create deployment web --image=nginx +kubectl expose deployment web --port=80 --type=LoadBalancer +kubectl get svc web -w +``` + +The service gets an `EXTERNAL-IP` from the simulator's public IP range +(`192.168.2.0/24`), and the corresponding rule shows up in CloudStack: + +```bash +curl -s "http://localhost:8080/client/api?command=listLoadBalancerRules&response=json..."; | jq +``` + +The CloudStack UI is also available: run the simulator with `-p 8081:5050` +and open <http://localhost:8081/>, logging in as `admin` / `password`. + +## What the scripts do + +`hack/e2e/up.sh` chains four numbered scripts. Each is independently runnable +and safe to re-run. All tunables live in `hack/e2e/env.sh` and can be +overridden from the environment. + +### 1. `10-simulator-up.sh` — simulator and zone + +Creates a docker bridge network (`cs-ccm-e2e`, `172.30.0.0/24`) that both the +simulator and the kind nodes will join, then starts the simulator on it: + +```bash +docker network create --subnet 172.30.0.0/24 cs-ccm-e2e +docker run -d --name cloudstack-simulator --network cs-ccm-e2e \ + -p 127.0.0.1:8080:8080 apache/cloudstack-simulator:4.22.1.0 +``` + +The image exposes three ports and it matters which one you use: + +| Port | What it is | +| --- | --- | +| **8080** | the management server API (`/client/api`) — **use this one** | +| 8096 | the unauthenticated integration API, used by marvin | +| 5050 | the Vue UI development server, which proxies to 8080 | + +The upstream simulator README suggests `-p 8080:5050`, which publishes the +*UI*. For API access, publish container port 8080 directly. + +Readiness is checked in three stages rather than with a fixed sleep: jetty +answering at all, then a successful `login`, then `listManagementServersMetrics` +returning a server. The last one matters because the CCM makes exactly that +call on startup and refuses to run until it succeeds. + +The zone is then deployed with marvin, which is preinstalled in the image: + +```bash +docker exec cloudstack-simulator python3 \ + /root/tools/marvin/marvin/deployDataCenter.py -i /root/setup/dev/advanced.cfg +``` + +This creates the `Sandbox-simulator` advanced zone with a public IP range of +`192.168.2.2`–`192.168.2.200`. + +Finally the script mints admin API keys. There is no signing involved: it logs +in with `admin`/`password`, which returns a `sessionkey` and a `JSESSIONID` +cookie, and then calls `getUserKeys`. Note that the sessionkey must be sent +both as a cookie *and* as a request parameter, and that `listUsers` is not a +substitute — it returns the API key but never the secret. `registerUserKeys` +is used only when no key pair exists yet, because it *rotates* the keys. + +Keys land in `hack/e2e/_out/keys.env`. + +### 2. `20-kind-up.sh` — the Kubernetes cluster + +```bash +KIND_EXPERIMENTAL_DOCKER_NETWORK=cs-ccm-e2e kind create cluster \ + --name cs-ccm-e2e --config hack/e2e/kind-config.yaml +``` + +The cluster config does two important things: + +* `cloud-provider: external` in every node's `kubeletExtraArgs`, so nodes + register with the `node.cloudprovider.kubernetes.io/uninitialized` taint. + Removing that taint is the CCM's job, and is how you know it works. +* `kubelet-preferred-address-types: InternalIP` on the API server. Once the + CCM initializes a node it sets the node's Hostname address to the CloudStack + instance's hostname, which for the simulator is the simulated hypervisor + agent and is not resolvable. Without this setting, `kubectl logs` and + `kubectl exec` stop working after node initialization. + +The cluster is named so that node names are deterministic: +`cs-ccm-e2e-control-plane`, `cs-ccm-e2e-worker`, `cs-ccm-e2e-worker2`. Two +workers exist so tests can check that the control plane node — which kubeadm +labels `node.kubernetes.io/exclude-from-external-load-balancers` — is left out +of load balancer membership. + +The script records each node's IP on the shared docker network into +`hack/e2e/_out/node-ips`. The next step depends on it. + +> CoreDNS stays `Pending` until the CCM removes the uninitialized taint. That +> is expected; don't wait for it. + +### 3. `30-topology-isolated.sh` — matching CloudStack VMs + +**This is the part that makes or breaks the environment.** The CCM looks up +each Kubernetes node by name in CloudStack, so a VM must exist whose name +exactly matches the node name. On top of that, kind starts kubelet with +`--node-ip=<the node's docker IP>`, and the CCM's node controller refuses to +initialize a node whose kubelet-reported IP is not among the addresses the +cloud provider reports for it. So the VMs must also carry the *same IP +addresses* as the kind node containers. + +The script therefore aligns the zone's guest CIDR with the docker subnet, +creates an isolated network on it, and deploys one VM per node pinned to that +node's IP: + +```bash +cs updateZone id=$ZONE guestcidraddress=172.30.0.0/24 +cs createNetwork name=ccm-e2e-iso networkofferingid=$OFFERING \ + gateway=172.30.0.1 netmask=255.255.255.0 zoneid=$ZONE +cs deployVirtualMachine name=cs-ccm-e2e-worker displayname=cs-ccm-e2e-worker \ + ipaddress=172.30.0.4 networkids=$NET ... +``` + +The offering used is `DefaultIsolatedNetworkOfferingWithSourceNatService`, +which provides the **Firewall** service — so on this network the CCM manages +firewall rules. (The VPC scenario below uses an offering that provides +**NetworkACL** instead, exercising the other branch.) + +All offerings and templates are looked up by name, because their UUIDs differ +between simulator deployments. + +### 4. `40-ccm-deploy.sh` — the controller + +Generates two `cloud-config` files that differ **only in `api-url`**: + +* `hack/e2e/_out/cloud-config` — used by the in-cluster deployment, pointing + at the simulator's IP on the `cs-ccm-e2e` docker network. +* `hack/e2e/_out/cloud-config-host` — used when you run the CCM as a host + process, pointing at `http://localhost:8080/client/api`. + +The in-cluster config must use the simulator's **IP address**, not its +container name or network alias: pods have their own network namespace and +cannot reach Docker's embedded DNS resolver, and `host.docker.internal` does +not exist on Linux Docker Engine. + +Both configs set `zone` explicitly. If `zone` is empty the CCM tries to +detect it by looking up its own pod, which cannot work when running as a host +process. + +The script then loads the image into kind, applies the repository's +[`deployment.yaml`](../deployment.yaml) and patches it for testing: the local +image with `imagePullPolicy: Never`, `--leader-elect=false` (single replica, +faster startup), `--v=4` for useful logs, and higher CPU limits — the stock +manifest's `limits.cpu: 50m` throttles informer startup badly on shared CI +runners. + +Finally it waits for every node to lose the uninitialized taint. + +## Running the CCM as a host process + +For interactive development and debugging, skip step 4 and run the binary +directly against the same environment: + +```bash +make +./cloudstack-ccm \ + --cloud-provider=external-cloudstack \ + --cloud-config=hack/e2e/_out/cloud-config-host \ + --kubeconfig=hack/e2e/_out/kubeconfig \ + --leader-elect=false \ + --v=4 +``` + +If the in-cluster CCM is already running, scale it down first so the two do +not fight over the same services: + +```bash +kubectl -n kube-system scale deployment/cloud-controller-manager --replicas=0 +``` + +### Debugging + +You can use the VS Code extension +[Go](https://marketplace.visualstudio.com/items?itemName=golang.go) to debug +the CCM. Add the following to `.vscode/launch.json`: + +```json +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Launch CloudStack CCM", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/cloudstack-ccm", + "env": {}, + "args": [ + "--cloud-provider=external-cloudstack", + "--cloud-config=${workspaceFolder}/hack/e2e/_out/cloud-config-host", + "--kubeconfig=${workspaceFolder}/hack/e2e/_out/kubeconfig", + "--leader-elect=false", + "--v=4" + ], + "showLog": true, + "trace": "verbose" + }, + { + "name": "Attach to Process", + "type": "go", + "request": "attach", + "mode": "local", + "processId": 0 + } + ] +} +``` + +To debug against a real CloudStack installation instead of the simulator, +point `--cloud-config` at your own `cloud-config` and `--kubeconfig` at your +cluster's kubeconfig. + +## The VPC scenario + +`hack/e2e/50-topology-vpc.sh` switches the environment to a VPC network so the +Network ACL code path can be exercised. It creates a VPC, a **custom** ACL list, +a tier network and per-node VMs, then re-points the CCM at them and restarts it. + +Two details are worth knowing: + +* The ACL list must be a custom one. The CCM deliberately refuses to add rules + to the built-in `default_allow` and `default_deny` lists. +* Everything is created inside a **CloudStack project**. The CCM matches VM + names across the whole account, and fails with `found hosts that belong to + different networks` if the matched VMs are spread over several networks. + Because CloudStack hides project resources from non-project queries and vice + versa, putting the VPC VMs in a project makes the two scenarios mutually + invisible without needing a second cluster or a second account. + +The tier reuses the same subnet as the isolated network, so the VMs keep the +same IP addresses and node initialization continues to work after the switch. + +## Running the tests + +Unit tests need nothing but Go: + +```bash +make test +``` + +The end-to-end suite needs the environment above. It is behind the `e2e` build +tag, so it never runs as part of `make test` or `go build ./...`: + +```bash +make test-e2e +``` + +Configuration comes from the environment, using the same variable names as the +existing opt-in acceptance tests in `cloudstack_test.go`: + +| Variable | Meaning | +| --- | --- | +| `KUBECONFIG` | cluster under test | +| `CS_API_URL` | CloudStack API endpoint as reachable from the test process | +| `CS_API_KEY`, `CS_SECRET_KEY` | CloudStack credentials | +| `CS_PROJECT_ID` | optional; set during the VPC phase | + +When any of them is missing the tests skip rather than fail. The same suite +runs against a real CloudStack installation — just point the variables at it. + +Each test creates its own namespace and cleans up after itself. Because load +balancer provisioning is asynchronous, all assertions poll rather than +assuming immediate consistency. + +### Known limitation: provider IDs + +kind starts kubelet with `--provider-id=kind://docker/<cluster>/<node>`, and +Kubernetes only allows a node's provider ID to be set once. In this +environment the CCM therefore never assigns the +`external-cloudstack://<instance UUID>` provider ID it would set on a real +cluster. `TestNode_ProviderID` detects this, logs the value it *would* have +assigned, and reports itself as skipped, so the gap stays visible instead of +quietly passing. Everything else about node initialization — taint removal, +labels, addresses — is exercised normally. + +## Continuous integration + +[`.github/workflows/e2e-simulator.yml`](../.github/workflows/e2e-simulator.yml) +runs this environment on every pull request and every push to `main`, as a +matrix of the latest two Kubernetes versions against the latest two CloudStack +releases. The CloudStack axis is not only version coverage: CloudStack 4.22 +and later update a load balancer rule's CIDR list in place, while earlier +versions delete and recreate the rule, so both branches get tested. + +All matrix cells run in parallel and a shared build job compiles the CCM image +once, so the whole workflow takes about as long as a single run — roughly +fifteen minutes, most of it the simulator image pull and zone deployment. + +To change the versions under test, edit the `k8s` and `acs` lists in the +matrix. Both use explicit patch-level tags +([`kindest/node`](https://hub.docker.com/r/kindest/node/tags) and +[`apache/cloudstack-simulator`](https://hub.docker.com/r/apache/cloudstack-simulator/tags)), +so a run is reproducible; avoid floating tags like `latest`. + +## Troubleshooting + +| Symptom | Cause | +| --- | --- | +| CCM exits with `no management servers found` | The account cannot call `listManagementServersMetrics`. This is a root-admin API; the default `User` role does not include it. | +| Nodes keep the uninitialized taint; CCM logs `provided node ip for node "..." is not valid` | The CloudStack VM's NIC IP does not match the IP kubelet registered with. Recreate the VM with `ipaddress=` set to the kind node's docker IP. | +| Services stay `<pending>`; CCM logs `none of the hosts matched the list of VMs retrieved from CS API` | No CloudStack VM has a name matching a Kubernetes node name. | +| CCM logs `found hosts that belong to different networks` | VMs matching the node names exist on more than one network — typically leftovers from a previous scenario. | +| No ACL rules are created on a VPC network | The tier uses `default_allow` or `default_deny`. The CCM only manages rules on custom ACL lists. | +| CoreDNS stuck `Pending` | Expected until the CCM removes the uninitialized taint. If it persists, the CCM is not working — check its logs. | +| `kubectl logs`/`exec` fail after nodes initialize | The API server is preferring the Hostname address, which the CCM set to the CloudStack instance hostname. Use `kubelet-preferred-address-types: InternalIP` as the provided kind config does. | +| Simulator never becomes ready | It runs `mvn jetty:run` and fetches from Maven Central at startup. Check `docker logs cloudstack-simulator`. | diff --git a/hack/e2e/10-simulator-up.sh b/hack/e2e/10-simulator-up.sh new file mode 100755 index 00000000..e475f8a4 --- /dev/null +++ b/hack/e2e/10-simulator-up.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Starts the CloudStack simulator, waits for it to be usable, deploys the +# advanced zone and mints admin API keys into _out/keys.env. + +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" +source "${E2E_ROOT}/lib/cs.sh" + +# --- docker network shared with kind ----------------------------------------- +if ! docker network inspect "$E2E_NET" >/dev/null 2>&1; then + log "creating docker network ${E2E_NET} (${E2E_SUBNET})" + docker network create --driver bridge --subnet "$E2E_SUBNET" --gateway "$E2E_GW" "$E2E_NET" +fi + +# --- simulator container ------------------------------------------------------ +if ! docker inspect "$SIM_NAME" >/dev/null 2>&1; then + log "starting simulator ${SIM_IMAGE} as ${SIM_NAME}" + # Container port 8080 is the management API (jetty); 5050 is only the UI + # dev server, which proxies to it. + docker run -d --name "$SIM_NAME" \ + --network "$E2E_NET" --network-alias cloudstack-simulator \ + -p "127.0.0.1:${SIM_HOST_PORT}:8080" \ + "$SIM_IMAGE" +else + log "simulator container ${SIM_NAME} already exists, reusing it" +fi + +# --- staged readiness --------------------------------------------------------- +jetty_up() { + local code + code="$(curl -s -o /dev/null -w '%{http_code}' -m 5 \ + "${CS_API_URL}?command=listCapabilities&response=json")" + [[ "$code" == "401" || "$code" == "200" ]] +} + +mgmt_server_up() { + local count + count="$(cs listManagementServersMetrics | jq -r '.listmanagementserversmetricsresponse.count // 0')" + [[ "$count" -ge 1 ]] +} + +wait_for 600 5 "jetty answering ${CS_API_URL}" jetty_up +wait_for 300 5 "CloudStack login as ${CS_ADMIN_USER}" cs_login +# The CCM calls listManagementServersMetrics at startup and refuses to run +# until it succeeds, so gate on the exact same call. +wait_for 300 5 "management server registered" mgmt_server_up + +# --- zone --------------------------------------------------------------------- +zone_enabled() { + local state + state="$(cs listZones "name=${ZONE_NAME}" | jq -r '.listzonesresponse.zone[0].allocationstate // empty')" + [[ "$state" == "Enabled" ]] +} + +host_up() { + local count + count="$(cs listHosts type=Routing state=Up | jq -r '.listhostsresponse.count // 0')" + [[ "$count" -ge 1 ]] +} + +if zone_enabled; then + log "zone ${ZONE_NAME} already deployed" +else + log "deploying zone ${ZONE_NAME} (this takes a few minutes)" + docker exec "$SIM_NAME" python3 /root/tools/marvin/marvin/deployDataCenter.py \ + -i /root/setup/dev/advanced.cfg +fi +wait_for 600 10 "zone ${ZONE_NAME} enabled" zone_enabled +wait_for 300 10 "at least one routing host up" host_up + +# --- admin API keys ----------------------------------------------------------- +admin_user_id="$(cs listUsers "username=${CS_ADMIN_USER}" | jq -r '.listusersresponse.user[0].id')" +[[ -n "$admin_user_id" && "$admin_user_id" != "null" ]] || die "could not find user ${CS_ADMIN_USER}" + +# getUserKeys first: registerUserKeys would rotate (invalidate) an existing +# pair, which is unfriendly to a long-lived local simulator. +keys="$(cs getUserKeys "id=${admin_user_id}")" +api_key="$(jq -r '.getuserkeysresponse.userkeys.apikey // empty' <<<"$keys")" +secret_key="$(jq -r '.getuserkeysresponse.userkeys.secretkey // empty' <<<"$keys")" +if [[ -z "$api_key" || -z "$secret_key" ]]; then + log "no existing keys, registering new ones" + keys="$(cs registerUserKeys "id=${admin_user_id}")" + api_key="$(jq -r '.registeruserkeysresponse.userkeys.apikey' <<<"$keys")" + secret_key="$(jq -r '.registeruserkeysresponse.userkeys.secretkey' <<<"$keys")" +fi +[[ -n "$api_key" && -n "$secret_key" ]] || die "failed to obtain admin API keys" + +cat >"${E2E_OUT}/keys.env" <<EOF +export CS_API_KEY='${api_key}' +export CS_SECRET_KEY='${secret_key}' +EOF +chmod 600 "${E2E_OUT}/keys.env" +log "simulator ready; admin API keys written to ${E2E_OUT}/keys.env" diff --git a/hack/e2e/20-kind-up.sh b/hack/e2e/20-kind-up.sh new file mode 100755 index 00000000..0b1bace9 --- /dev/null +++ b/hack/e2e/20-kind-up.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Creates the kind cluster on the shared docker network and records the +# node-name -> docker-IP map that the CloudStack VMs must reproduce. + +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" + +if kind get clusters 2>/dev/null | grep -qx "$KIND_CLUSTER"; then + log "kind cluster ${KIND_CLUSTER} already exists, reusing it" +else + log "creating kind cluster ${KIND_CLUSTER} (image ${KIND_NODE_IMAGE}) on network ${E2E_NET}" + KIND_EXPERIMENTAL_DOCKER_NETWORK="$E2E_NET" kind create cluster \ + --name "$KIND_CLUSTER" \ + --config "${E2E_ROOT}/kind-config.yaml" \ + --image "$KIND_NODE_IMAGE" \ + --wait 180s +fi + +kind get kubeconfig --name "$KIND_CLUSTER" >"${E2E_OUT}/kubeconfig" +chmod 600 "${E2E_OUT}/kubeconfig" + +# The CCM only initializes a node when the CloudStack VM's NIC IP matches the +# IP kubelet registered with (kind passes --node-ip). Record each node's IP on +# the shared network so 30-topology-* can pin the VMs to them. +: >"${E2E_OUT}/node-ips" +for node in $(kind get nodes --name "$KIND_CLUSTER"); do + ip="$(docker inspect -f "{{(index .NetworkSettings.Networks \"${E2E_NET}\").IPAddress}}" "$node")" + [[ -n "$ip" ]] || die "could not determine IP of ${node} on ${E2E_NET}" + echo "${node} ${ip}" >>"${E2E_OUT}/node-ips" +done +log "node IPs:" +cat "${E2E_OUT}/node-ips" >&2 + +log "kind cluster ready; kubeconfig at ${E2E_OUT}/kubeconfig" diff --git a/hack/e2e/30-topology-isolated.sh b/hack/e2e/30-topology-isolated.sh new file mode 100755 index 00000000..6a560bcd --- /dev/null +++ b/hack/e2e/30-topology-isolated.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Creates an isolated guest network matching the kind docker subnet and +# deploys one CloudStack VM per kind node, pinned to the node's docker IP. + +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" +source "${E2E_ROOT}/lib/cs.sh" + +[[ -s "${E2E_OUT}/node-ips" ]] || die "missing ${E2E_OUT}/node-ips — run 20-kind-up.sh first" +cs_login || die "CloudStack login failed" + +zone_id="$(cs listZones "name=${ZONE_NAME}" | jq -r '.listzonesresponse.zone[0].id')" +[[ -n "$zone_id" && "$zone_id" != "null" ]] || die "zone ${ZONE_NAME} not found" + +# The stock zone guest CIDR is 10.1.1.0/24; align it with the docker subnet so +# createNetwork accepts our gateway/netmask. +cs updateZone "id=${zone_id}" "guestcidraddress=${E2E_SUBNET}" >/dev/null + +net_id="$(cs listNetworks "keyword=${E2E_ISO_NETWORK}" listall=true | + jq -r --arg n "$E2E_ISO_NETWORK" '.listnetworksresponse.network[]? | select(.name == $n) | .id')" +if [[ -z "$net_id" ]]; then + offering_id="$(cs listNetworkOfferings name=DefaultIsolatedNetworkOfferingWithSourceNatService state=Enabled | + jq -r '.listnetworkofferingsresponse.networkoffering[0].id')" + [[ -n "$offering_id" && "$offering_id" != "null" ]] || die "isolated network offering not found" + log "creating isolated network ${E2E_ISO_NETWORK} (${E2E_SUBNET})" + net_id="$(cs_async createNetwork "name=${E2E_ISO_NETWORK}" "displaytext=${E2E_ISO_NETWORK}" \ + "zoneid=${zone_id}" "networkofferingid=${offering_id}" \ + "gateway=${E2E_GW}" "netmask=${E2E_NETMASK}" | + jq -r '.network.id // .id')" +fi +[[ -n "$net_id" && "$net_id" != "null" ]] || die "failed to create network ${E2E_ISO_NETWORK}" + +offering_id="$(cs listServiceOfferings "name=${E2E_SERVICE_OFFERING}" | + jq -r '.listserviceofferingsresponse.serviceoffering[0].id')" +# templatefilter=executable excludes the SYSTEM (router) template, which +# cannot be used to deploy user VMs. +template_id="$(cs listTemplates templatefilter=executable "zoneid=${zone_id}" hypervisor=Simulator | + jq -r '.listtemplatesresponse.template[]? | select(.isready == true) | .id' | head -1)" +[[ -n "$offering_id" && "$offering_id" != "null" ]] || die "service offering '${E2E_SERVICE_OFFERING}' not found" +[[ -n "$template_id" ]] || die "no ready simulator template found" + +while read -r node ip; do + existing="$(cs listVirtualMachines "keyword=${node}" listall=true | + jq -r --arg n "$node" '.listvirtualmachinesresponse.virtualmachine[]? | select(.name == $n) | .id')" + if [[ -n "$existing" ]]; then + log "VM ${node} already exists" + continue + fi + log "deploying VM ${node} with IP ${ip}" + cs_async deployVirtualMachine "name=${node}" "displayname=${node}" \ + "zoneid=${zone_id}" "serviceofferingid=${offering_id}" "templateid=${template_id}" \ + "networkids=${net_id}" "ipaddress=${ip}" "startvm=true" >/dev/null +done <"${E2E_OUT}/node-ips" + +# Post-condition: every node must now have a matching VM on the right IP, or +# the CCM will never initialize that node. +while read -r node ip; do + vm_ip="$(cs listVirtualMachines "keyword=${node}" listall=true | + jq -r --arg n "$node" '.listvirtualmachinesresponse.virtualmachine[]? | select(.name == $n) | .nic[0].ipaddress')" + [[ "$vm_ip" == "$ip" ]] || die "VM ${node} has IP '${vm_ip}', expected ${ip}" +done <"${E2E_OUT}/node-ips" + +cat >"${E2E_OUT}/ids.env" <<EOF +export E2E_ZONE_ID='${zone_id}' +export E2E_ISO_NETWORK_ID='${net_id}' +export E2E_SERVICE_OFFERING_ID='${offering_id}' +export E2E_TEMPLATE_ID='${template_id}' +EOF +log "isolated topology ready (network ${net_id})" diff --git a/hack/e2e/40-ccm-deploy.sh b/hack/e2e/40-ccm-deploy.sh new file mode 100755 index 00000000..71f39ab0 --- /dev/null +++ b/hack/e2e/40-ccm-deploy.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generates cloud-config files, loads the CCM image into kind, deploys +# deployment.yaml and waits until all nodes are initialized. + +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" + +# shellcheck source=/dev/null +source "${E2E_OUT}/keys.env" 2>/dev/null || die "missing ${E2E_OUT}/keys.env — run 10-simulator-up.sh first" + +# The CCM pod must reach the simulator via its IP on the shared docker +# network: pods cannot resolve docker's embedded DNS, and host.docker.internal +# does not exist on Linux. +sim_ip="$(docker inspect -f "{{(index .NetworkSettings.Networks \"${E2E_NET}\").IPAddress}}" "$SIM_NAME")" +[[ -n "$sim_ip" ]] || die "could not determine simulator IP on ${E2E_NET}" + +# PROJECT_ID is optional; 50-topology-vpc.sh re-runs this script with it set. +project_line="" +if [[ -n "${E2E_PROJECT_ID:-}" ]]; then + project_line="project-id = ${E2E_PROJECT_ID}" +fi + +# In-cluster and host-process configs differ only in api-url. +cat >"${E2E_OUT}/cloud-config" <<EOF +[Global] +api-url = http://${sim_ip}:8080/client/api +api-key = ${CS_API_KEY} +secret-key = ${CS_SECRET_KEY} +zone = ${ZONE_NAME} +region = ${E2E_REGION} +${project_line} +EOF +sed "s|http://${sim_ip}:8080|http://localhost:${SIM_HOST_PORT}|" \ + "${E2E_OUT}/cloud-config" >"${E2E_OUT}/cloud-config-host" +chmod 600 "${E2E_OUT}/cloud-config" "${E2E_OUT}/cloud-config-host" + +# Build the image if it is not present (CI loads a prebuilt artifact instead). +if ! docker image inspect "$CCM_IMAGE" >/dev/null 2>&1; then + log "building ${CCM_IMAGE}" + docker build -t "$CCM_IMAGE" "$REPO_ROOT" +fi +kind load docker-image "$CCM_IMAGE" --name "$KIND_CLUSTER" + +kubectl -n kube-system create secret generic cloudstack-secret \ + --from-file=cloud-config="${E2E_OUT}/cloud-config" \ + --dry-run=client -o yaml | kubectl apply -f - + +kubectl apply -f "${REPO_ROOT}/deployment.yaml" +# Adjust the stock manifest for e2e: local image, no leader election (single +# replica, faster startup), verbose logs, and enough CPU that informer startup +# is not throttled on shared runners. +kubectl -n kube-system patch deployment cloud-controller-manager --type=json -p '[ + {"op":"replace","path":"/spec/template/spec/containers/0/image","value":"'"$CCM_IMAGE"'"}, + {"op":"replace","path":"/spec/template/spec/containers/0/imagePullPolicy","value":"Never"}, + {"op":"replace","path":"/spec/template/spec/containers/0/args","value":[ + "--cloud-provider=external-cloudstack","--cloud-config=/config/cloud-config", + "--leader-elect=false","--v=4"]}, + {"op":"replace","path":"/spec/template/spec/containers/0/resources","value":{ + "requests":{"cpu":"100m","memory":"128Mi"},"limits":{"cpu":"1","memory":"512Mi"}}} +]' + +kubectl -n kube-system rollout status deployment/cloud-controller-manager --timeout=180s + +nodes_initialized() { + local taints + taints="$(kubectl get nodes -o jsonpath='{.items[*].spec.taints[?(@.key=="node.cloudprovider.kubernetes.io/uninitialized")].key}')" + [[ -z "$taints" ]] +} +# If this times out, check the CCM log for +# 'provided node ip for node ... is not valid': it means the CloudStack VM's +# NIC IP does not match the kind node's docker IP. +wait_for 300 5 "all nodes initialized by the CCM" nodes_initialized + +log "CCM deployed and all nodes initialized" +kubectl get nodes -o wide >&2 diff --git a/hack/e2e/50-topology-vpc.sh b/hack/e2e/50-topology-vpc.sh new file mode 100755 index 00000000..117e4979 --- /dev/null +++ b/hack/e2e/50-topology-vpc.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Phase 2: creates a CloudStack project containing a VPC, a custom ACL list, a +# tier network and per-node VMs, then re-points the CCM at the project. +# +# A project is used so the VPC VMs and the phase-1 isolated-network VMs are +# mutually invisible: the CCM's verifyHosts matches VM names account-wide and +# fails when matched VMs are on different networks. With project-id set, only +# project resources are visible. The tier reuses the docker subnet, so the VMs +# get the same IPs as phase 1 and node initialization keeps working. + +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" +source "${E2E_ROOT}/lib/cs.sh" + +[[ -s "${E2E_OUT}/node-ips" ]] || die "missing ${E2E_OUT}/node-ips — run 20-kind-up.sh first" +# shellcheck source=/dev/null +source "${E2E_OUT}/ids.env" 2>/dev/null || die "missing ${E2E_OUT}/ids.env — run 30-topology-isolated.sh first" +cs_login || die "CloudStack login failed" + +# Phase-1 LoadBalancer services must be gone before the CCM switches projects, +# or their CloudStack resources leak (the project-scoped CCM can't see them). +leftover="$(kubectl get svc -A -o jsonpath='{range .items[?(@.spec.type=="LoadBalancer")]}{.metadata.namespace}/{.metadata.name}{"\n"}{end}')" +if [[ -n "$leftover" ]]; then + log "deleting leftover LoadBalancer services:" + echo "$leftover" >&2 + while IFS=/ read -r ns name; do + kubectl -n "$ns" delete svc "$name" --wait=true --timeout=120s + done <<<"$leftover" +fi + +project_id="$(cs listProjects listall=true "name=${E2E_PROJECT}" | + jq -r '.listprojectsresponse.project[0].id // empty')" +if [[ -z "$project_id" ]]; then + log "creating project ${E2E_PROJECT}" + project_id="$(cs_async createProject "name=${E2E_PROJECT}" "displaytext=${E2E_PROJECT}" | + jq -r '.project.id // .id')" +fi +[[ -n "$project_id" && "$project_id" != "null" ]] || die "failed to create project" + +vpc_id="$(cs listVPCs listall=true "projectid=${project_id}" "name=${E2E_VPC}" | + jq -r '.listvpcsresponse.vpc[0].id // empty')" +if [[ -z "$vpc_id" ]]; then + vpc_offering_id="$(cs listVPCOfferings "name=Default VPC offering" | + jq -r '.listvpcofferingsresponse.vpcoffering[0].id')" + log "creating VPC ${E2E_VPC} (${E2E_VPC_CIDR})" + vpc_id="$(cs_async createVPC "name=${E2E_VPC}" "displaytext=${E2E_VPC}" \ + "zoneid=${E2E_ZONE_ID}" "cidr=${E2E_VPC_CIDR}" \ + "vpcofferingid=${vpc_offering_id}" "projectid=${project_id}" | + jq -r '.vpc.id // .id')" +fi +[[ -n "$vpc_id" && "$vpc_id" != "null" ]] || die "failed to create VPC" + +# A custom ACL list: the CCM refuses to manage rules on the built-in +# default_allow / default_deny lists. +acl_id="$(cs listNetworkACLLists "vpcid=${vpc_id}" "name=${E2E_ACL_LIST}" | + jq -r '.listnetworkacllistsresponse.networkacllist[0].id // empty')" +if [[ -z "$acl_id" ]]; then + log "creating ACL list ${E2E_ACL_LIST}" + acl_id="$(cs_async createNetworkACLList "name=${E2E_ACL_LIST}" \ + "description=${E2E_ACL_LIST}" "vpcid=${vpc_id}" | + jq -r '.networkacllist.id // .id')" +fi +[[ -n "$acl_id" && "$acl_id" != "null" ]] || die "failed to create ACL list" + +tier_id="$(cs listNetworks listall=true "projectid=${project_id}" "keyword=${E2E_TIER}" | + jq -r --arg n "$E2E_TIER" '.listnetworksresponse.network[]? | select(.name == $n) | .id')" +if [[ -z "$tier_id" ]]; then + tier_offering_id="$(cs listNetworkOfferings name=DefaultIsolatedNetworkOfferingForVpcNetworks state=Enabled | + jq -r '.listnetworkofferingsresponse.networkoffering[0].id')" + log "creating VPC tier ${E2E_TIER} (${E2E_SUBNET})" + tier_id="$(cs_async createNetwork "name=${E2E_TIER}" "displaytext=${E2E_TIER}" \ + "zoneid=${E2E_ZONE_ID}" "networkofferingid=${tier_offering_id}" \ + "vpcid=${vpc_id}" "aclid=${acl_id}" \ + "gateway=${E2E_GW}" "netmask=${E2E_NETMASK}" "projectid=${project_id}" | + jq -r '.network.id // .id')" +fi +[[ -n "$tier_id" && "$tier_id" != "null" ]] || die "failed to create VPC tier" + +while read -r node ip; do + existing="$(cs listVirtualMachines listall=true "projectid=${project_id}" "keyword=${node}" | + jq -r --arg n "$node" '.listvirtualmachinesresponse.virtualmachine[]? | select(.name == $n) | .id')" + if [[ -n "$existing" ]]; then + log "project VM ${node} already exists" + continue + fi + log "deploying project VM ${node} with IP ${ip}" + cs_async deployVirtualMachine "name=${node}" "displayname=${node}" \ + "zoneid=${E2E_ZONE_ID}" "serviceofferingid=${E2E_SERVICE_OFFERING_ID}" \ + "templateid=${E2E_TEMPLATE_ID}" "networkids=${tier_id}" \ + "ipaddress=${ip}" "projectid=${project_id}" "startvm=true" >/dev/null +done <"${E2E_OUT}/node-ips" + +{ + echo "export E2E_PROJECT_ID='${project_id}'" + echo "export E2E_VPC_ID='${vpc_id}'" + echo "export E2E_ACL_ID='${acl_id}'" + echo "export E2E_TIER_ID='${tier_id}'" +} >>"${E2E_OUT}/ids.env" + +# Re-point the CCM at the project and restart it. +E2E_PROJECT_ID="$project_id" "${E2E_ROOT}/40-ccm-deploy.sh" +kubectl -n kube-system rollout restart deployment/cloud-controller-manager +kubectl -n kube-system rollout status deployment/cloud-controller-manager --timeout=180s + +log "VPC topology ready (project ${project_id}, tier ${tier_id}, acl ${acl_id})" diff --git a/hack/e2e/90-collect-artifacts.sh b/hack/e2e/90-collect-artifacts.sh new file mode 100755 index 00000000..99610b17 --- /dev/null +++ b/hack/e2e/90-collect-artifacts.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Collects debugging artifacts from the simulator, the kind cluster and the +# CloudStack API into _out/artifacts. Never fails. + +set -uo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" +source "${E2E_ROOT}/lib/cs.sh" + +ART="${E2E_OUT}/artifacts" +mkdir -p "$ART" + +log "collecting artifacts into ${ART}" + +docker logs "$SIM_NAME" >"${ART}/simulator.log" 2>&1 + +# kubectl logs may stop working once the CCM rewrites node addresses, so fall +# back to reading container logs on the control-plane node directly. +if ! kubectl -n kube-system logs deployment/cloud-controller-manager --tail=-1 \ + >"${ART}/ccm.log" 2>&1; then + docker exec "${KIND_CLUSTER}-control-plane" bash -c \ + 'crictl ps -a --name cloud-controller-manager -q | head -1 | xargs -r crictl logs' \ + >"${ART}/ccm.log" 2>&1 +fi + +kubectl get nodes -o yaml >"${ART}/nodes.yaml" 2>&1 +kubectl get svc -A -o yaml >"${ART}/services.yaml" 2>&1 +kubectl describe svc -A >"${ART}/svc-describe.txt" 2>&1 +kubectl get events -A --sort-by=.lastTimestamp >"${ART}/events.txt" 2>&1 +kubectl -n kube-system get pods -o wide >"${ART}/kube-system-pods.txt" 2>&1 + +if cs_login; then + # Dump each resource twice: without a project (the isolated-network phase) + # and with projectid=-1, which for an admin spans all projects (the VPC + # phase). Otherwise the VPC phase's resources are invisible here. + for cmd in listLoadBalancerRules listPublicIpAddresses listFirewallRules \ + listNetworkACLs listVirtualMachines listNetworks; do + name="cs-$(echo "$cmd" | tr '[:upper:]' '[:lower:]')" + cs "$cmd" listall=true | jq . >"${ART}/${name}.json" 2>&1 + cs "$cmd" listall=true projectid=-1 | jq . >"${ART}/${name}-projects.json" 2>&1 + done +fi + +kind export logs "${ART}/kind" --name "$KIND_CLUSTER" >/dev/null 2>&1 + +log "artifacts collected" +exit 0 diff --git a/hack/e2e/99-down.sh b/hack/e2e/99-down.sh new file mode 100755 index 00000000..682aa86f --- /dev/null +++ b/hack/e2e/99-down.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Tears down everything the harness created. + +set -uo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/env.sh" +source "${E2E_ROOT}/lib/log.sh" + +log "deleting kind cluster ${KIND_CLUSTER}" +kind delete cluster --name "$KIND_CLUSTER" 2>/dev/null + +log "removing simulator container ${SIM_NAME}" +docker rm -f "$SIM_NAME" 2>/dev/null + +log "removing docker network ${E2E_NET}" +docker network rm "$E2E_NET" 2>/dev/null + +rm -f "${E2E_OUT}/keys.env" "${E2E_OUT}/cloud-config" "${E2E_OUT}/cloud-config-host" \ + "${E2E_OUT}/kubeconfig" "${E2E_OUT}/node-ips" "${E2E_OUT}/ids.env" + +log "done" +exit 0 diff --git a/hack/e2e/env.sh b/hack/e2e/env.sh new file mode 100755 index 00000000..bb8f1f2f --- /dev/null +++ b/hack/e2e/env.sh @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# All tunables for the simulator e2e harness in one place. +# Every value can be overridden from the environment (CI does this for the +# simulator tag and the kind node image). + +E2E_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export E2E_ROOT +export REPO_ROOT="${E2E_ROOT}/../.." +export E2E_OUT="${E2E_ROOT}/_out" + +# Docker network shared by the kind nodes and the simulator. The subnet must +# match the isolated network created in CloudStack: the CCM refuses to +# initialize a node whose kubelet-reported IP is missing from the CloudStack +# VM's NICs, so the VMs are deployed with the kind nodes' docker IPs. +export E2E_NET="${E2E_NET:-cs-ccm-e2e}" +export E2E_SUBNET="${E2E_SUBNET:-172.30.0.0/24}" +export E2E_GW="${E2E_GW:-172.30.0.1}" +export E2E_NETMASK="${E2E_NETMASK:-255.255.255.0}" + +# CloudStack simulator +export SIM_NAME="${SIM_NAME:-cloudstack-simulator}" +export SIM_TAG="${SIM_TAG:-4.22.1.0}" +export SIM_IMAGE="${SIM_IMAGE:-apache/cloudstack-simulator:${SIM_TAG}}" +export SIM_HOST_PORT="${SIM_HOST_PORT:-8080}" +export CS_API_URL="${CS_API_URL:-http://localhost:${SIM_HOST_PORT}/client/api}"; +export CS_ADMIN_USER="${CS_ADMIN_USER:-admin}" +export CS_ADMIN_PASS="${CS_ADMIN_PASS:-password}" +export ZONE_NAME="${ZONE_NAME:-Sandbox-simulator}" +export E2E_REGION="${E2E_REGION:-simulator-region}" + +# kind +export KIND_CLUSTER="${KIND_CLUSTER:-cs-ccm-e2e}" +export KIND_NODE_IMAGE="${KIND_NODE_IMAGE:-kindest/node:v1.37.0}" + +# CCM image built from this checkout +export CCM_IMAGE="${CCM_IMAGE:-apache/cloudstack-kubernetes-provider:e2e}" + +# CloudStack names created by the harness +export E2E_ISO_NETWORK="${E2E_ISO_NETWORK:-ccm-e2e-iso}" +export E2E_PROJECT="${E2E_PROJECT:-ccm-e2e-vpc}" +export E2E_VPC="${E2E_VPC:-ccm-e2e-vpc}" +export E2E_VPC_CIDR="${E2E_VPC_CIDR:-172.30.0.0/22}" +export E2E_ACL_LIST="${E2E_ACL_LIST:-ccm-e2e-acl}" +export E2E_TIER="${E2E_TIER:-ccm-e2e-tier}" +export E2E_SERVICE_OFFERING="${E2E_SERVICE_OFFERING:-Small Instance}" + +export KUBECONFIG="${KUBECONFIG:-${E2E_OUT}/kubeconfig}" + +mkdir -p "${E2E_OUT}" diff --git a/hack/e2e/kind-config.yaml b/hack/e2e/kind-config.yaml new file mode 100644 index 00000000..5c79c241 --- /dev/null +++ b/hack/e2e/kind-config.yaml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# kind cluster for the CloudStack simulator e2e environment. +# +# - `cloud-provider: external` makes every node register with the +# node.cloudprovider.kubernetes.io/uninitialized taint, which the CCM under +# test is responsible for removing. +# - `kubelet-preferred-address-types: InternalIP` keeps `kubectl logs`/`exec` +# working after node initialization: the CCM sets the node's Hostname +# address to the CloudStack instance hostname (the simulated hypervisor +# agent), which is not resolvable from the API server. +# - Two workers so tests can assert that the control-plane node (labeled +# node.kubernetes.io/exclude-from-external-load-balancers by kubeadm) is +# excluded from load balancer membership. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: cs-ccm-e2e +networking: + ipFamily: ipv4 + apiServerAddress: "127.0.0.1" +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + - | + kind: ClusterConfiguration + apiServer: + extraArgs: + kubelet-preferred-address-types: InternalIP + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + - role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external diff --git a/hack/e2e/lib/cs.sh b/hack/e2e/lib/cs.sh new file mode 100644 index 00000000..8c8f6a63 --- /dev/null +++ b/hack/e2e/lib/cs.sh @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Minimal CloudStack API client for the harness. Uses session (login) +# authentication so no request signing is needed. The sessionkey must be sent +# both as a cookie and as a request parameter. + +CS_JSESSIONID="" +CS_SESSIONKEY="" + +cs_login() { + local out + out="$(curl -sS -i -m 30 -X POST "$CS_API_URL" \ + --data-urlencode "command=login" \ + --data-urlencode "username=${CS_ADMIN_USER}" \ + --data-urlencode "password=${CS_ADMIN_PASS}" \ + --data-urlencode "domain=/" \ + --data-urlencode "response=json")" || return 1 + CS_JSESSIONID="$(sed -n 's/.*JSESSIONID=\([^;]*\);.*/\1/p' <<<"$out" | head -1)" + CS_SESSIONKEY="$(sed -n 's/.*"sessionkey":"\([^"]*\)".*/\1/p' <<<"$out")" + [[ -n "$CS_SESSIONKEY" && -n "$CS_JSESSIONID" ]] +} + +# cs <command> [key=value ...] -> JSON response on stdout +cs() { + local cmd="$1" + shift + local args=(--data-urlencode "command=${cmd}" + --data-urlencode "response=json" + --data-urlencode "sessionkey=${CS_SESSIONKEY}") + local kv + for kv in "$@"; do + args+=(--data-urlencode "$kv") + done + curl -sS -G -m 120 \ + -b "JSESSIONID=${CS_JSESSIONID}; sessionkey=${CS_SESSIONKEY}" \ + "${args[@]}" "$CS_API_URL" +} + +# cs_async <command> [key=value ...] -> jobresult JSON on stdout. +# Transparently handles synchronous commands (no jobid in the response). +cs_async() { + local resp jobid result status i errtext + resp="$(cs "$@")" || return 1 + # An API error carries no jobid, so it would otherwise be mistaken for a + # synchronous success. Check for it explicitly. + errtext="$(jq -r '.[].errortext // empty' <<<"$resp" 2>/dev/null)" + if [[ -n "$errtext" ]]; then + echo "CloudStack API error from $1: ${errtext}" >&2 + return 1 + fi + jobid="$(jq -r '.[].jobid // empty' <<<"$resp" 2>/dev/null)" + if [[ -z "$jobid" ]]; then + # Synchronous command: unwrap the single top-level response object. + jq '.[]' <<<"$resp" + return 0 + fi + for ((i = 0; i < 120; i++)); do + result="$(cs queryAsyncJobResult "jobid=${jobid}")" || return 1 + status="$(jq -r '.queryasyncjobresultresponse.jobstatus' <<<"$result")" + case "$status" in + 1) + jq '.queryasyncjobresultresponse.jobresult' <<<"$result" + return 0 + ;; + 2) + echo "async job ${jobid} failed: $(jq -c '.queryasyncjobresultresponse.jobresult' <<<"$result")" >&2 + return 1 + ;; + esac + sleep 5 + done + echo "async job ${jobid} did not finish in time" >&2 + return 1 +} diff --git a/hack/e2e/lib/log.sh b/hack/e2e/lib/log.sh new file mode 100644 index 00000000..3bf5bd2f --- /dev/null +++ b/hack/e2e/lib/log.sh @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +log() { + echo "[$(date -u +%H:%M:%S)] $*" >&2 +} + +die() { + log "FATAL: $*" + exit 1 +} + +# wait_for <timeout-seconds> <interval-seconds> <description> <command...> +# Polls <command...> until it succeeds or the timeout elapses. +wait_for() { + local timeout=$1 interval=$2 desc=$3 + shift 3 + local start=$SECONDS + log "waiting up to ${timeout}s for: ${desc}" + while ((SECONDS - start < timeout)); do + if "$@" >/dev/null 2>&1; then + log "ready after $((SECONDS - start))s: ${desc}" + return 0 + fi + sleep "$interval" + done + die "timed out after ${timeout}s waiting for: ${desc}" +} + +# retry <attempts> <sleep-seconds> <command...> +retry() { + local attempts=$1 pause=$2 i + shift 2 + for ((i = 1; i <= attempts; i++)); do + if "$@"; then + return 0 + fi + log "attempt ${i}/${attempts} failed: $*" + sleep "$pause" + done + return 1 +} diff --git a/hack/e2e/up.sh b/hack/e2e/up.sh new file mode 100755 index 00000000..9f050c90 --- /dev/null +++ b/hack/e2e/up.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# One-shot bring-up of the full simulated environment: +# simulator + zone -> kind cluster -> CloudStack VMs -> CCM. + +set -euo pipefail +here="$(dirname "${BASH_SOURCE[0]}")" + +"${here}/10-simulator-up.sh" +"${here}/20-kind-up.sh" +"${here}/30-topology-isolated.sh" +"${here}/40-ccm-deploy.sh" + +echo +echo "Environment is up. Try it:" +echo " export KUBECONFIG=${here}/_out/kubeconfig" +echo " kubectl create deployment web --image=nginx" +echo " kubectl expose deployment web --port=80 --type=LoadBalancer" +echo " kubectl get svc web -w # EXTERNAL-IP appears from 192.168.2.0/24" +echo +echo "Run the e2e suite: make test-e2e" +echo "Tear down: ${here}/99-down.sh" diff --git a/test/e2e/annotations_test.go b/test/e2e/annotations_test.go new file mode 100644 index 00000000..67cc27fe --- /dev/null +++ b/test/e2e/annotations_test.go @@ -0,0 +1,176 @@ +//go:build e2e + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package e2e + +import ( + "context" + "strings" + "testing" + + "github.com/blang/semver/v4" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + annotationSourceCidrs = "service.beta.kubernetes.io/cloudstack-load-balancer-source-cidrs" + annotationHostname = "service.beta.kubernetes.io/cloudstack-load-balancer-hostname" + annotationIPAssociated = "service.beta.kubernetes.io/cloudstack-load-balancer-ip-associated-by-controller" //nolint:gosec +) + +func TestAnnot_SourceCIDRs(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(func(s *corev1.Service) { + s.Annotations = map[string]string{ + annotationSourceCidrs: "10.0.0.0/8,192.168.100.0/24", + } + }) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 1) + for _, cidr := range []string{"10.0.0.0/8", "192.168.100.0/24"} { + if !strings.Contains(rules[0].Cidrlist, cidr) { + t.Errorf("rule cidrlist = %q, want it to contain %s", rules[0].Cidrlist, cidr) + } + } + originalRuleID := rules[0].Id + + // Change the CIDR list. On >= 4.22 the rule is updated in place (same + // ID); on older versions it is deleted and recreated (new ID). + f.UpdateService(svc, func(s *corev1.Service) { + s.Annotations[annotationSourceCidrs] = "172.16.0.0/12" + }) + inPlace := f.Version.GTE(semver.Version{Major: 4, Minor: 22, Patch: 0}) + f.Eventually(lbSyncTimeout, lbSyncInterval, "cidr list update to propagate", + func() (bool, error) { + current, err := f.LBRules(lbName) + if err != nil || len(current) != 1 { + return false, err + } + if !strings.Contains(current[0].Cidrlist, "172.16.0.0/12") { + return false, nil + } + if inPlace && current[0].Id != originalRuleID { + t.Errorf("expected in-place cidr update on %s (rule ID changed %s -> %s)", + f.Version, originalRuleID, current[0].Id) + } + if !inPlace && current[0].Id == originalRuleID { + t.Errorf("expected rule recreation on %s (rule ID unchanged)", f.Version) + } + return true, nil + }) +} + +func TestAnnot_Hostname(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(func(s *corev1.Service) { + s.Annotations = map[string]string{ + annotationHostname: "lb.example.com", + } + }) + + ingress := f.WaitForIngressIP(svc) + if ingress.Hostname != "lb.example.com" { + t.Errorf("ingress hostname = %q, want lb.example.com", ingress.Hostname) + } + if ingress.IP != "" { + t.Errorf("ingress IP = %q, want empty when hostname annotation is set", ingress.IP) + } +} + +func TestAnnot_SessionAffinity(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(func(s *corev1.Service) { + s.Spec.SessionAffinity = corev1.ServiceAffinityClientIP + }) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 1) + if rules[0].Algorithm != "source" { + t.Errorf("algorithm = %q, want source for sessionAffinity ClientIP", rules[0].Algorithm) + } + + f.UpdateService(svc, func(s *corev1.Service) { + s.Spec.SessionAffinity = corev1.ServiceAffinityNone + }) + f.Eventually(lbSyncTimeout, lbSyncInterval, "algorithm to revert to roundrobin", + func() (bool, error) { + current, err := f.LBRules(lbName) + if err != nil || len(current) != 1 { + return false, err + } + return current[0].Algorithm == "roundrobin", nil + }) +} + +func TestAnnot_ExplicitLoadBalancerIP(t *testing.T) { + f := NewFramework(t) + + // Pick a free IP from the simulator's public range instead of hardcoding + // one that a parallel test may have grabbed. + p := f.CS.Address.NewListPublicIpAddressesParams() + p.SetAllocatedonly(false) + p.SetListall(true) + p.SetState("Free") + resp, err := f.CS.Address.ListPublicIpAddresses(p) + if err != nil || resp.Count == 0 { + t.Fatalf("listing free public IPs: count=%d err=%v", resp.Count, err) + } + freeIP := resp.PublicIpAddresses[0].Ipaddress + + svc := f.CreateLBService(func(s *corev1.Service) { + s.Spec.LoadBalancerIP = freeIP + }) + + ingress := f.WaitForIngressIP(svc) + if ingress.IP != freeIP { + t.Fatalf("ingress IP = %q, want requested %q", ingress.IP, freeIP) + } + + // The controller associated the IP itself, so it must record that fact + // on the service; on deletion the IP must be released again. + f.Eventually(lbSyncTimeout, lbSyncInterval, "ip-associated-by-controller annotation", + func() (bool, error) { + current, err := f.K8s.CoreV1().Services(svc.Namespace).Get( + context.Background(), svc.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return current.Annotations[annotationIPAssociated] == "true", nil + }) + + f.DeleteServiceAndWait(svc) + f.Eventually(lbSyncTimeout, lbSyncInterval, "explicitly requested IP to be released", + func() (bool, error) { + p := f.CS.Address.NewListPublicIpAddressesParams() + p.SetIpaddress(freeIP) + p.SetAllocatedonly(false) + p.SetListall(true) + resp, err := f.CS.Address.ListPublicIpAddresses(p) + if err != nil || resp.Count == 0 { + return false, err + } + return resp.PublicIpAddresses[0].Allocated == "", nil + }) +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go new file mode 100644 index 00000000..cf82b3f2 --- /dev/null +++ b/test/e2e/framework.go @@ -0,0 +1,381 @@ +//go:build e2e + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Package e2e contains end-to-end tests that run against a live Kubernetes +// cluster whose cloud-controller-manager talks to a CloudStack management +// server (normally the simulator brought up by hack/e2e/up.sh). +// +// Configuration comes from the environment: +// +// KUBECONFIG kubeconfig of the cluster under test +// CS_API_URL CloudStack API endpoint (as reachable from the test process) +// CS_API_KEY CloudStack API key +// CS_SECRET_KEY CloudStack secret key +// CS_PROJECT_ID optional project scoping (set for the VPC phase) +// +// When any required variable is missing, the tests skip. +package e2e + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver/v4" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + lbSyncTimeout = 3 * time.Minute + lbSyncInterval = 3 * time.Second +) + +// Framework bundles the clients and helpers shared by all e2e tests. +type Framework struct { + T *testing.T + K8s kubernetes.Interface + CS *cloudstack.CloudStackClient + Namespace string + ProjectID string + Version semver.Version +} + +// NewFramework builds clients from the environment, skipping the test when +// the environment is not configured. It creates a per-test namespace that is +// deleted on cleanup. +func NewFramework(t *testing.T) *Framework { + t.Helper() + + apiURL := os.Getenv("CS_API_URL") + apiKey := os.Getenv("CS_API_KEY") + secretKey := os.Getenv("CS_SECRET_KEY") + if apiURL == "" || apiKey == "" || secretKey == "" { + t.Skip("CS_API_URL/CS_API_KEY/CS_SECRET_KEY not set; skipping e2e test") + } + + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + t.Skip("KUBECONFIG not set; skipping e2e test") + } + restCfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + t.Fatalf("building kubeconfig: %v", err) + } + k8s, err := kubernetes.NewForConfig(restCfg) + if err != nil { + t.Fatalf("building kubernetes client: %v", err) + } + + verifySSL := true + if noVerify, err := strconv.ParseBool(os.Getenv("CS_SSL_NO_VERIFY")); err == nil { + verifySSL = !noVerify + } + cs := cloudstack.NewAsyncClient(apiURL, apiKey, secretKey, verifySSL) + + f := &Framework{ + T: t, + K8s: k8s, + CS: cs, + ProjectID: os.Getenv("CS_PROJECT_ID"), + } + f.Version = f.managementServerVersion() + f.Namespace = f.createNamespace() + return f +} + +func (f *Framework) managementServerVersion() semver.Version { + f.T.Helper() + resp, err := f.CS.Management.ListManagementServersMetrics( + f.CS.Management.NewListManagementServersMetricsParams()) + if err != nil { + f.T.Fatalf("listing management servers: %v", err) + } + if resp.Count == 0 { + f.T.Fatal("no management servers found") + } + raw := strings.Join(strings.SplitN(resp.ManagementServersMetrics[0].Version, ".", 4)[0:3], ".") + v, err := semver.ParseTolerant(raw) + if err != nil { + f.T.Fatalf("parsing management server version %q: %v", raw, err) + } + return v +} + +func (f *Framework) createNamespace() string { + f.T.Helper() + buf := make([]byte, 4) + if _, err := rand.Read(buf); err != nil { + f.T.Fatalf("generating namespace suffix: %v", err) + } + name := "ccm-e2e-" + hex.EncodeToString(buf) + _, err := f.K8s.CoreV1().Namespaces().Create(context.Background(), + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: name}}, metav1.CreateOptions{}) + if err != nil { + f.T.Fatalf("creating namespace %s: %v", name, err) + } + f.T.Cleanup(func() { + _ = f.K8s.CoreV1().Namespaces().Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + return name +} + +// Eventually polls cond until it returns true or the timeout elapses. +func (f *Framework) Eventually(timeout, interval time.Duration, desc string, cond func() (bool, error)) { + f.T.Helper() + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + ok, err := cond() + lastErr = err + if ok { + return + } + time.Sleep(interval) + } + f.T.Fatalf("timed out after %s waiting for %s (last error: %v)", timeout, desc, lastErr) +} + +// CreateLBService creates a LoadBalancer service in the test namespace and +// registers cleanup that both deletes it and waits for the CloudStack rules +// to disappear, so a leaked rule cannot poison later tests. +func (f *Framework) CreateLBService(mutate func(*corev1.Service)) *corev1.Service { + f.T.Helper() + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e", + Namespace: f.Namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{"app": "e2e"}, + Ports: []corev1.ServicePort{ + {Name: "http", Port: 80, Protocol: corev1.ProtocolTCP}, + }, + }, + } + if mutate != nil { + mutate(svc) + } + created, err := f.K8s.CoreV1().Services(f.Namespace).Create( + context.Background(), svc, metav1.CreateOptions{}) + if err != nil { + f.T.Fatalf("creating service: %v", err) + } + f.T.Cleanup(func() { f.DeleteServiceAndWait(created) }) + return created +} + +// DeleteServiceAndWait deletes the service (if it still exists) and waits for +// its CloudStack load balancer rules to be cleaned up. +func (f *Framework) DeleteServiceAndWait(svc *corev1.Service) { + f.T.Helper() + err := f.K8s.CoreV1().Services(svc.Namespace).Delete( + context.Background(), svc.Name, metav1.DeleteOptions{}) + if err != nil { + return // already gone + } + lbName := defaultLoadBalancerName(svc) + deadline := time.Now().Add(lbSyncTimeout) + for time.Now().Before(deadline) { + rules, err := f.LBRules(lbName) + if err == nil && len(rules) == 0 { + return + } + time.Sleep(lbSyncInterval) + } + f.T.Logf("warning: load balancer rules for %s not cleaned up in time", lbName) +} + +// defaultLoadBalancerName mirrors cloudprovider.DefaultLoadBalancerName: "a" +// followed by the service UID with dashes stripped, truncated to 32 chars. +func defaultLoadBalancerName(svc *corev1.Service) string { + name := "a" + strings.ReplaceAll(string(svc.UID), "-", "") + if len(name) > 32 { + name = name[:32] + } + return name +} + +// LBRules returns the CloudStack load balancer rules whose names start with +// the given LB name. +func (f *Framework) LBRules(lbName string) ([]*cloudstack.LoadBalancerRule, error) { + p := f.CS.LoadBalancer.NewListLoadBalancerRulesParams() + p.SetKeyword(lbName) + p.SetListall(true) + if f.ProjectID != "" { + p.SetProjectid(f.ProjectID) + } + resp, err := f.CS.LoadBalancer.ListLoadBalancerRules(p) + if err != nil { + return nil, err + } + var rules []*cloudstack.LoadBalancerRule + for _, r := range resp.LoadBalancerRules { + if strings.HasPrefix(r.Name, lbName) { + rules = append(rules, r) + } + } + return rules, nil +} + +// WaitForIngressIP waits until the service has a load balancer ingress entry +// and returns it. +func (f *Framework) WaitForIngressIP(svc *corev1.Service) corev1.LoadBalancerIngress { + f.T.Helper() + var ingress corev1.LoadBalancerIngress + f.Eventually(lbSyncTimeout, lbSyncInterval, + fmt.Sprintf("service %s/%s to get an ingress address", svc.Namespace, svc.Name), + func() (bool, error) { + current, err := f.K8s.CoreV1().Services(svc.Namespace).Get( + context.Background(), svc.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(current.Status.LoadBalancer.Ingress) == 0 { + return false, nil + } + ingress = current.Status.LoadBalancer.Ingress[0] + return true, nil + }) + return ingress +} + +// WaitForLBRules waits until exactly want rules exist for lbName and returns them. +func (f *Framework) WaitForLBRules(lbName string, want int) []*cloudstack.LoadBalancerRule { + f.T.Helper() + var rules []*cloudstack.LoadBalancerRule + f.Eventually(lbSyncTimeout, lbSyncInterval, + fmt.Sprintf("%d load balancer rule(s) named %s-*", want, lbName), + func() (bool, error) { + var err error + rules, err = f.LBRules(lbName) + if err != nil { + return false, err + } + return len(rules) == want, nil + }) + return rules +} + +// FirewallRules lists the firewall rules on a public IP. +func (f *Framework) FirewallRules(publicIPID string) ([]*cloudstack.FirewallRule, error) { + p := f.CS.Firewall.NewListFirewallRulesParams() + p.SetIpaddressid(publicIPID) + p.SetListall(true) + if f.ProjectID != "" { + p.SetProjectid(f.ProjectID) + } + resp, err := f.CS.Firewall.ListFirewallRules(p) + if err != nil { + return nil, err + } + return resp.FirewallRules, nil +} + +// ACLRules lists the network ACL rules on an ACL list. +func (f *Framework) ACLRules(aclListID string) ([]*cloudstack.NetworkACL, error) { + p := f.CS.NetworkACL.NewListNetworkACLsParams() + p.SetAclid(aclListID) + p.SetListall(true) + if f.ProjectID != "" { + p.SetProjectid(f.ProjectID) + } + resp, err := f.CS.NetworkACL.ListNetworkACLs(p) + if err != nil { + return nil, err + } + return resp.NetworkACLs, nil +} + +// PublicIP fetches a public IP address record by its ID. +func (f *Framework) PublicIP(id string) (*cloudstack.PublicIpAddress, error) { + p := f.CS.Address.NewListPublicIpAddressesParams() + p.SetId(id) + p.SetListall(true) + p.SetAllocatedonly(false) + if f.ProjectID != "" { + p.SetProjectid(f.ProjectID) + } + resp, err := f.CS.Address.ListPublicIpAddresses(p) + if err != nil { + return nil, err + } + if resp.Count == 0 { + return nil, nil + } + return resp.PublicIpAddresses[0], nil +} + +// VMByName returns the CloudStack VM with the given name, or nil. +func (f *Framework) VMByName(name string) (*cloudstack.VirtualMachine, error) { + vm, count, err := f.CS.VirtualMachine.GetVirtualMachineByName( + name, cloudstack.WithProject(f.ProjectID)) + if err != nil { + if count == 0 { + return nil, nil + } + return nil, err + } + return vm, nil +} + +// Nodes returns all nodes of the cluster under test. +func (f *Framework) Nodes() []corev1.Node { + f.T.Helper() + nodes, err := f.K8s.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{}) + if err != nil { + f.T.Fatalf("listing nodes: %v", err) + } + return nodes.Items +} + +// UpdateService applies mutate to the latest version of the service and +// updates it, retrying on conflicts. +func (f *Framework) UpdateService(svc *corev1.Service, mutate func(*corev1.Service)) *corev1.Service { + f.T.Helper() + var updated *corev1.Service + f.Eventually(30*time.Second, time.Second, "service update to apply", + func() (bool, error) { + current, err := f.K8s.CoreV1().Services(svc.Namespace).Get( + context.Background(), svc.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + mutate(current) + updated, err = f.K8s.CoreV1().Services(svc.Namespace).Update( + context.Background(), current, metav1.UpdateOptions{}) + if err != nil { + return false, err + } + return true, nil + }) + return updated +} diff --git a/test/e2e/loadbalancer_test.go b/test/e2e/loadbalancer_test.go new file mode 100644 index 00000000..c2eaae1a --- /dev/null +++ b/test/e2e/loadbalancer_test.go @@ -0,0 +1,213 @@ +//go:build e2e + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package e2e + +import ( + "context" + "fmt" + "net" + "strconv" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLB_CreateSingleTCPPort(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(nil) + lbName := defaultLoadBalancerName(svc) + + ingress := f.WaitForIngressIP(svc) + if ingress.IP == "" { + t.Fatalf("expected an ingress IP, got %+v", ingress) + } + if ip := net.ParseIP(ingress.IP); ip == nil { + t.Fatalf("ingress IP %q is not a valid IP", ingress.IP) + } + + rules := f.WaitForLBRules(lbName, 1) + rule := rules[0] + wantName := fmt.Sprintf("%s-tcp-80", lbName) + if rule.Name != wantName { + t.Errorf("rule name = %q, want %q", rule.Name, wantName) + } + if rule.Algorithm != "roundrobin" { + t.Errorf("rule algorithm = %q, want roundrobin", rule.Algorithm) + } + if rule.Publicport != "80" { + t.Errorf("rule public port = %q, want 80", rule.Publicport) + } + // The private port must be the service's NodePort. + current, err := f.K8s.CoreV1().Services(svc.Namespace).Get(context.Background(), svc.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("getting service: %v", err) + } + nodePort := strconv.Itoa(int(current.Spec.Ports[0].NodePort)) + if rule.Privateport != nodePort { + t.Errorf("rule private port = %q, want NodePort %q", rule.Privateport, nodePort) + } + if rule.Publicip != ingress.IP { + t.Errorf("rule public IP = %q, want ingress IP %q", rule.Publicip, ingress.IP) + } + if !strings.Contains(rule.Cidrlist, "0.0.0.0/0") { + t.Errorf("rule cidrlist = %q, want it to contain 0.0.0.0/0", rule.Cidrlist) + } + + // The isolated network offering supports the Firewall service, so a + // firewall rule must exist for the port. + f.Eventually(lbSyncTimeout, lbSyncInterval, "firewall rule for port 80", + func() (bool, error) { + fwRules, err := f.FirewallRules(rule.Publicipid) + if err != nil { + return false, err + } + for _, fw := range fwRules { + if fw.Startport == 80 && fw.Endport == 80 && strings.EqualFold(fw.Protocol, "tcp") { + return true, nil + } + } + return false, nil + }) +} + +func TestLB_MultiPort(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(func(s *corev1.Service) { + s.Spec.Ports = []corev1.ServicePort{ + {Name: "http", Port: 80, Protocol: corev1.ProtocolTCP}, + {Name: "https", Port: 443, Protocol: corev1.ProtocolTCP}, + } + }) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 2) + if rules[0].Publicipid != rules[1].Publicipid { + t.Errorf("expected both rules to share a public IP, got %q and %q", + rules[0].Publicipid, rules[1].Publicipid) + } + ports := map[string]bool{} + for _, r := range rules { + ports[r.Publicport] = true + } + if !ports["80"] || !ports["443"] { + t.Errorf("expected rules for ports 80 and 443, got %v", ports) + } +} + +func TestLB_NodeMembership(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(nil) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 1) + + // Only schedulable workers participate; kubeadm labels the control plane + // node.kubernetes.io/exclude-from-external-load-balancers. + wantIDs := map[string]bool{} + for _, node := range f.Nodes() { + if _, excluded := node.Labels["node.kubernetes.io/exclude-from-external-load-balancers"]; excluded { + continue + } + vm, err := f.VMByName(node.Name) + if err != nil || vm == nil { + t.Fatalf("looking up VM for node %s: %v", node.Name, err) + } + wantIDs[vm.Id] = true + } + if len(wantIDs) == 0 { + t.Fatal("no candidate worker nodes found") + } + + f.Eventually(lbSyncTimeout, lbSyncInterval, "load balancer rule instances to match worker VMs", + func() (bool, error) { + p := f.CS.LoadBalancer.NewListLoadBalancerRuleInstancesParams(rules[0].Id) + resp, err := f.CS.LoadBalancer.ListLoadBalancerRuleInstances(p) + if err != nil { + return false, err + } + gotIDs := map[string]bool{} + for _, inst := range resp.LoadBalancerRuleInstances { + gotIDs[inst.Id] = true + } + if len(gotIDs) != len(wantIDs) { + return false, fmt.Errorf("got %d instances, want %d", len(gotIDs), len(wantIDs)) + } + for id := range wantIDs { + if !gotIDs[id] { + return false, fmt.Errorf("VM %s missing from rule instances", id) + } + } + return true, nil + }) +} + +func TestLB_PortChange(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(nil) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + f.WaitForLBRules(lbName, 1) + + f.UpdateService(svc, func(s *corev1.Service) { + s.Spec.Ports[0].Port = 8080 + }) + + // The old rule (port 80) must be replaced by a new one (port 8080). + f.Eventually(lbSyncTimeout, lbSyncInterval, "rule for port 8080 to replace port 80", + func() (bool, error) { + rules, err := f.LBRules(lbName) + if err != nil { + return false, err + } + return len(rules) == 1 && rules[0].Publicport == "8080", nil + }) +} + +func TestLB_Delete(t *testing.T) { + f := NewFramework(t) + svc := f.CreateLBService(nil) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 1) + publicIPID := rules[0].Publicipid + + f.DeleteServiceAndWait(svc) + + if remaining, err := f.LBRules(lbName); err != nil || len(remaining) != 0 { + t.Errorf("expected no remaining rules, got %d (err %v)", len(remaining), err) + } + // The auto-allocated public IP must be released. + f.Eventually(lbSyncTimeout, lbSyncInterval, "public IP to be released", + func() (bool, error) { + ip, err := f.PublicIP(publicIPID) + if err != nil { + return false, err + } + return ip == nil || ip.Allocated == "", nil + }) +} diff --git a/test/e2e/node_test.go b/test/e2e/node_test.go new file mode 100644 index 00000000..e94b00c6 --- /dev/null +++ b/test/e2e/node_test.go @@ -0,0 +1,137 @@ +//go:build e2e + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package e2e + +import ( + "os" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +const providerIDPrefix = "external-cloudstack://" + +// TestNode_Initialized asserts the CCM removed the uninitialized taint from +// every node. +func TestNode_Initialized(t *testing.T) { + f := NewFramework(t) + for _, node := range f.Nodes() { + for _, taint := range node.Spec.Taints { + if taint.Key == "node.cloudprovider.kubernetes.io/uninitialized" { + t.Errorf("node %s still has the uninitialized taint", node.Name) + } + } + } +} + +// TestNode_ProviderID asserts every node's providerID references the matching +// CloudStack VM. +// +// kind starts kubelet with --provider-id=kind://..., and Kubernetes only lets +// the provider ID be set once, so under the kind-based harness the CCM never +// gets to assign it. Where that is the case the test verifies instead that +// the CCM would derive the right value, and reports the node as skipped so +// the limitation stays visible rather than silently reducing coverage. +func TestNode_ProviderID(t *testing.T) { + f := NewFramework(t) + checked := 0 + for _, node := range f.Nodes() { + vm, err := f.VMByName(node.Name) + if err != nil { + t.Fatalf("looking up VM for node %s: %v", node.Name, err) + } + if vm == nil { + t.Fatalf("no CloudStack VM named %s", node.Name) + } + want := providerIDPrefix + vm.Id + + if node.Spec.ProviderID != "" && !strings.HasPrefix(node.Spec.ProviderID, providerIDPrefix) { + t.Logf("node %s has a foreign provider ID %q (set by the infrastructure, "+ + "not the CCM); expected CloudStack provider ID would be %q", + node.Name, node.Spec.ProviderID, want) + continue + } + if node.Spec.ProviderID != want { + t.Errorf("node %s providerID = %q, want %q", node.Name, node.Spec.ProviderID, want) + } + checked++ + } + if checked == 0 { + t.Skip("every node has a provider ID assigned by the infrastructure; " + + "the CCM's provider ID assignment is not exercised by this environment") + } +} + +// TestNode_Labels asserts the CCM applied instance-type, zone and region +// labels from CloudStack metadata. +func TestNode_Labels(t *testing.T) { + f := NewFramework(t) + region := os.Getenv("E2E_REGION") + if region == "" { + region = "simulator-region" + } + for _, node := range f.Nodes() { + vm, err := f.VMByName(node.Name) + if err != nil || vm == nil { + t.Fatalf("looking up VM for node %s: %v", node.Name, err) + } + // labelInvalidCharsRegex strips characters that are invalid in label + // values, e.g. "Small Instance" becomes "SmallInstance". + if got := node.Labels[corev1.LabelInstanceTypeStable]; got == "" { + t.Errorf("node %s is missing label %s", node.Name, corev1.LabelInstanceTypeStable) + } + if got := node.Labels[corev1.LabelTopologyZone]; got != vm.Zonename { + t.Errorf("node %s zone label = %q, want %q", node.Name, got, vm.Zonename) + } + if got := node.Labels[corev1.LabelTopologyRegion]; got != region { + t.Errorf("node %s region label = %q, want %q", node.Name, got, region) + } + } +} + +// TestNode_InternalIP asserts each node's InternalIP equals its CloudStack +// VM's NIC address. This is the contract that makes the whole environment +// work: kubelet registers with the docker IP, and the CCM only initializes +// the node because the VM reports the same address. +func TestNode_InternalIP(t *testing.T) { + f := NewFramework(t) + for _, node := range f.Nodes() { + vm, err := f.VMByName(node.Name) + if err != nil || vm == nil { + t.Fatalf("looking up VM for node %s: %v", node.Name, err) + } + if len(vm.Nic) == 0 { + t.Fatalf("VM %s has no NICs", node.Name) + } + var internalIP string + for _, addr := range node.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + internalIP = addr.Address + } + } + if internalIP != vm.Nic[0].Ipaddress { + t.Errorf("node %s InternalIP = %q, want VM NIC IP %q", + node.Name, internalIP, vm.Nic[0].Ipaddress) + } + } +} diff --git a/test/e2e/vpc_test.go b/test/e2e/vpc_test.go new file mode 100644 index 00000000..69033ecd --- /dev/null +++ b/test/e2e/vpc_test.go @@ -0,0 +1,132 @@ +//go:build e2e + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package e2e + +import ( + "os" + "strings" + "testing" +) + +// vpcFramework skips unless the harness is in the VPC phase +// (50-topology-vpc.sh exports E2E_ACL_ID / E2E_VPC_ID and CS_PROJECT_ID). +func vpcFramework(t *testing.T) (*Framework, string, string) { + t.Helper() + aclID := os.Getenv("E2E_ACL_ID") + vpcID := os.Getenv("E2E_VPC_ID") + if aclID == "" || vpcID == "" || os.Getenv("CS_PROJECT_ID") == "" { + t.Skip("E2E_ACL_ID/E2E_VPC_ID/CS_PROJECT_ID not set; skipping VPC phase test") + } + return NewFramework(t), aclID, vpcID +} + +// TestVPC_LoadBalancer covers the VPC path end to end: the LB rule is +// created, the public IP is associated with the VPC, ingress traffic is +// allowed via a Network ACL rule on the custom ACL list (not a firewall +// rule), and everything is cleaned up on delete. +func TestVPC_LoadBalancer(t *testing.T) { + f, aclID, vpcID := vpcFramework(t) + + svc := f.CreateLBService(nil) + lbName := defaultLoadBalancerName(svc) + + f.WaitForIngressIP(svc) + rules := f.WaitForLBRules(lbName, 1) + rule := rules[0] + + ip, err := f.PublicIP(rule.Publicipid) + if err != nil || ip == nil { + t.Fatalf("fetching public IP %s: %v", rule.Publicipid, err) + } + if ip.Vpcid != vpcID { + t.Errorf("public IP vpcid = %q, want %q", ip.Vpcid, vpcID) + } + + // An ACL rule for the port must appear on the custom ACL list. + f.Eventually(lbSyncTimeout, lbSyncInterval, "network ACL rule for port 80", + func() (bool, error) { + aclRules, err := f.ACLRules(aclID) + if err != nil { + return false, err + } + for _, r := range aclRules { + if r.Startport == "80" && r.Endport == "80" && + strings.EqualFold(r.Protocol, "tcp") && + strings.EqualFold(r.Action, "Allow") && + strings.EqualFold(r.Traffictype, "Ingress") { + return true, nil + } + } + return false, nil + }) + + // The VPC tier offering has no Firewall service, so no firewall rule may + // be created for the port. + fwRules, err := f.FirewallRules(rule.Publicipid) + if err != nil { + t.Fatalf("listing firewall rules: %v", err) + } + for _, fw := range fwRules { + if fw.Startport == 80 && fw.Endport == 80 { + t.Errorf("unexpected firewall rule on VPC public IP: %+v", fw) + } + } + + // Deleting the service must remove the ACL rule again. + f.DeleteServiceAndWait(svc) + f.Eventually(lbSyncTimeout, lbSyncInterval, "network ACL rule to be removed", + func() (bool, error) { + aclRules, err := f.ACLRules(aclID) + if err != nil { + return false, err + } + for _, r := range aclRules { + if r.Startport == "80" && r.Endport == "80" && strings.EqualFold(r.Protocol, "tcp") { + return false, nil + } + } + return true, nil + }) +} + +// TestVPC_NodesReinitialized asserts the CCM re-initialized the nodes against +// the project VMs after the phase switch. +func TestVPC_NodesReinitialized(t *testing.T) { + f, _, _ := vpcFramework(t) + for _, node := range f.Nodes() { + vm, err := f.VMByName(node.Name) + if err != nil { + t.Fatalf("looking up project VM for node %s: %v", node.Name, err) + } + if vm == nil { + t.Errorf("no project VM named %s visible with CS_PROJECT_ID", node.Name) + } + } + // No node may carry the uninitialized taint after the CCM restart. + for _, node := range f.Nodes() { + for _, taint := range node.Spec.Taints { + if taint.Key == "node.cloudprovider.kubernetes.io/uninitialized" { + t.Errorf("node %s still has the uninitialized taint", node.Name) + } + } + } +}
