DaanHoogland commented on code in PR #12386: URL: https://github.com/apache/cloudstack/pull/12386#discussion_r2704493753
########## test/integration/component/test_kubernetes_cluster_affinity_groups.py: ########## @@ -0,0 +1,940 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Tests for Kubernetes cluster affinity groups feature""" + +import unittest +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (listInfrastructure, + listKubernetesSupportedVersions, + addKubernetesSupportedVersion, + deleteKubernetesSupportedVersion, + listKubernetesClusters, + createKubernetesCluster, + stopKubernetesCluster, + startKubernetesCluster, + deleteKubernetesCluster, + scaleKubernetesCluster, + destroyVirtualMachine, + deleteNetwork) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import (ServiceOffering, + Account, + AffinityGroup, + Configurations) +from marvin.lib.utils import (cleanup_resources, + random_gen) +from marvin.lib.common import (get_zone, + get_domain) +from marvin.sshClient import SshClient +from nose.plugins.attrib import attr +from marvin.lib.decoratorGenerators import skipTestIf + +import time + +_multiprocess_shared_ = True + +RAND_SUFFIX = random_gen() + + +class TestKubernetesClusterAffinityGroups(cloudstackTestCase): + """ + Tests for CKS Affinity Groups feature (since 4.23.0) + + This feature allows specifying different affinity groups for each + Kubernetes node type (CONTROL, WORKER, ETCD). + """ + + @classmethod + def setUpClass(cls): + testClient = super(TestKubernetesClusterAffinityGroups, cls).getClsTestClient() + if testClient is None: + raise unittest.SkipTest("Marvin test client not available - check marvin configuration") + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.hypervisor = testClient.getHypervisorInfo() + cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ + + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + cls.hypervisorNotSupported = True + + cls.setup_failed = False + cls._cleanup = [] + cls.kubernetes_version_ids = [] + cls.initial_configuration_cks_enabled = None + + cls.k8s_version = cls.services.get("cks_kubernetes_version_upgrade_to", + cls.services.get("cks_kubernetes_version_upgrade_from")) + + if cls.hypervisorNotSupported == False: + cls.endpoint_url = Configurations.list(cls.apiclient, name="endpoint.url")[0].value + if "localhost" in cls.endpoint_url: + endpoint_url = "http://%s:%d/client/api"; % (cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"]) + cls.debug("Setting endpoint.url to %s" % endpoint_url) + Configurations.update(cls.apiclient, "endpoint.url", endpoint_url) + + cls.initial_configuration_cks_enabled = Configurations.list( + cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") + Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") + cls.restartServer() + + cls.cks_service_offering = None + + if cls.setup_failed == False: + try: + cls.kubernetes_version = cls.addKubernetesSupportedVersion( + cls.services["cks_kubernetes_versions"][cls.k8s_version]) + cls.kubernetes_version_ids.append(cls.kubernetes_version.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state: %s" % e) + + if cls.setup_failed == False: + cks_offering_data = cls.services["cks_service_offering"] + cks_offering_data["name"] = 'CKS-Instance-' + random_gen() + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cls._cleanup.append(cls.cks_service_offering) + + cls.domain = get_domain(cls.apiclient) + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + cls.default_network = None + + return + + @classmethod + def tearDownClass(cls): + # Delete added Kubernetes supported version + for version_id in cls.kubernetes_version_ids: + try: + cls.deleteKubernetesSupportedVersion(version_id) + except Exception as e: + cls.debug("Error during cleanup for Kubernetes versions: %s" % e) + + try: + # Restore CKS enabled + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Restoring Kubernetes Service enabled value") + Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") + cls.restartServer() + + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + + return + + @classmethod + def restartServer(cls): + """Restart management server""" + cls.debug("Restarting management server") + sshClient = SshClient( + cls.mgtSvrDetails["mgtSvrIp"], + 22, + cls.mgtSvrDetails["user"], + cls.mgtSvrDetails["passwd"] + ) + command = "service cloudstack-management stop" + sshClient.execute(command) + + command = "service cloudstack-management start" + sshClient.execute(command) + + # Wait for management to come up in 5 mins + timeout = time.time() + 300 + while time.time() < timeout: + if cls.isManagementUp() is True: + return + time.sleep(5) + cls.setup_failed = True + cls.debug("Management server did not come up, failing") + return + + @classmethod + def isManagementUp(cls): + try: + cls.apiclient.listInfrastructure(listInfrastructure.listInfrastructureCmd()) + return True + except Exception: + return False + + @classmethod + def waitForKubernetesSupportedVersionIsoReadyState(cls, version_id, retries=30, interval=60): + """Check if Kubernetes supported version ISO is in Ready state""" + while retries > 0: + time.sleep(interval) + list_versions_response = cls.listKubernetesSupportedVersion(version_id) + if not hasattr(list_versions_response, 'isostate') or not list_versions_response or not list_versions_response.isostate: + retries = retries - 1 + continue + if 'Ready' == list_versions_response.isostate: + return + elif 'Failed' == list_versions_response.isostate: + raise Exception("Failed to download template: status - %s" % list_versions_response.isostate) + retries = retries - 1 + raise Exception("Kubernetes supported version Ready state timed out") + + @classmethod + def listKubernetesSupportedVersion(cls, version_id): + listKubernetesSupportedVersionsCmd = listKubernetesSupportedVersions.listKubernetesSupportedVersionsCmd() + listKubernetesSupportedVersionsCmd.id = version_id + versionResponse = cls.apiclient.listKubernetesSupportedVersions(listKubernetesSupportedVersionsCmd) + return versionResponse[0] + + @classmethod + def addKubernetesSupportedVersion(cls, version_service): + addKubernetesSupportedVersionCmd = addKubernetesSupportedVersion.addKubernetesSupportedVersionCmd() + addKubernetesSupportedVersionCmd.semanticversion = version_service["semanticversion"] + addKubernetesSupportedVersionCmd.name = 'v' + version_service["semanticversion"] + '-' + random_gen() + addKubernetesSupportedVersionCmd.url = version_service["url"] + addKubernetesSupportedVersionCmd.mincpunumber = version_service["mincpunumber"] + addKubernetesSupportedVersionCmd.minmemory = version_service["minmemory"] + kubernetes_version = cls.apiclient.addKubernetesSupportedVersion(addKubernetesSupportedVersionCmd) + cls.debug("Waiting for Kubernetes version with ID %s to be ready" % kubernetes_version.id) + cls.waitForKubernetesSupportedVersionIsoReadyState(kubernetes_version.id) + kubernetes_version = cls.listKubernetesSupportedVersion(kubernetes_version.id) + return kubernetes_version + + @classmethod + def deleteKubernetesSupportedVersion(cls, version_id): + deleteKubernetesSupportedVersionCmd = deleteKubernetesSupportedVersion.deleteKubernetesSupportedVersionCmd() + deleteKubernetesSupportedVersionCmd.id = version_id + cls.apiclient.deleteKubernetesSupportedVersion(deleteKubernetesSupportedVersionCmd) + + @classmethod + def listKubernetesCluster(cls, cluster_id=None, cluster_name=None): + listKubernetesClustersCmd = listKubernetesClusters.listKubernetesClustersCmd() + listKubernetesClustersCmd.listall = True + if cluster_id is not None: + listKubernetesClustersCmd.id = cluster_id + if cluster_name is not None: + listKubernetesClustersCmd.name = cluster_name + clusterResponse = cls.apiclient.listKubernetesClusters(listKubernetesClustersCmd) + if (cluster_id is not None or cluster_name is not None) and clusterResponse is not None: + return clusterResponse[0] + return clusterResponse + + @classmethod + def deleteKubernetesCluster(cls, cluster_id): + deleteKubernetesClusterCmd = deleteKubernetesCluster.deleteKubernetesClusterCmd() + deleteKubernetesClusterCmd.id = cluster_id + response = cls.apiclient.deleteKubernetesCluster(deleteKubernetesClusterCmd) + return response + + @classmethod + def stopKubernetesCluster(cls, cluster_id): + stopKubernetesClusterCmd = stopKubernetesCluster.stopKubernetesClusterCmd() + stopKubernetesClusterCmd.id = cluster_id + response = cls.apiclient.stopKubernetesCluster(stopKubernetesClusterCmd) + return response + + def setUp(self): + self.services = self.testClient.getParsedTestDataConfig() + self.apiclient = self.testClient.getApiClient() + self.dbclient = self.testClient.getDbConnection() + self.cleanup = [] + self.aff_grp = [] + return + + def tearDown(self): + try: + cleanup_resources(self.apiclient, self.cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) + return Review Comment: ```suggestion def tearDown(self): super(TestKubernetesClusterAffinityGroups, self).tearDown() ``` ########## test/integration/component/test_kubernetes_cluster_affinity_groups.py: ########## @@ -0,0 +1,940 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Tests for Kubernetes cluster affinity groups feature""" + +import unittest +from marvin.cloudstackTestCase import cloudstackTestCase +from marvin.cloudstackAPI import (listInfrastructure, + listKubernetesSupportedVersions, + addKubernetesSupportedVersion, + deleteKubernetesSupportedVersion, + listKubernetesClusters, + createKubernetesCluster, + stopKubernetesCluster, + startKubernetesCluster, + deleteKubernetesCluster, + scaleKubernetesCluster, + destroyVirtualMachine, + deleteNetwork) +from marvin.cloudstackException import CloudstackAPIException +from marvin.lib.base import (ServiceOffering, + Account, + AffinityGroup, + Configurations) +from marvin.lib.utils import (cleanup_resources, + random_gen) +from marvin.lib.common import (get_zone, + get_domain) +from marvin.sshClient import SshClient +from nose.plugins.attrib import attr +from marvin.lib.decoratorGenerators import skipTestIf + +import time + +_multiprocess_shared_ = True + +RAND_SUFFIX = random_gen() + + +class TestKubernetesClusterAffinityGroups(cloudstackTestCase): + """ + Tests for CKS Affinity Groups feature (since 4.23.0) + + This feature allows specifying different affinity groups for each + Kubernetes node type (CONTROL, WORKER, ETCD). + """ + + @classmethod + def setUpClass(cls): + testClient = super(TestKubernetesClusterAffinityGroups, cls).getClsTestClient() + if testClient is None: + raise unittest.SkipTest("Marvin test client not available - check marvin configuration") + cls.apiclient = testClient.getApiClient() + cls.services = testClient.getParsedTestDataConfig() + cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests()) + cls.hypervisor = testClient.getHypervisorInfo() + cls.mgtSvrDetails = cls.config.__dict__["mgtSvr"][0].__dict__ + + cls.hypervisorNotSupported = False + if cls.hypervisor.lower() not in ["kvm", "vmware", "xenserver"]: + cls.hypervisorNotSupported = True + + cls.setup_failed = False + cls._cleanup = [] + cls.kubernetes_version_ids = [] + cls.initial_configuration_cks_enabled = None + + cls.k8s_version = cls.services.get("cks_kubernetes_version_upgrade_to", + cls.services.get("cks_kubernetes_version_upgrade_from")) + + if cls.hypervisorNotSupported == False: + cls.endpoint_url = Configurations.list(cls.apiclient, name="endpoint.url")[0].value + if "localhost" in cls.endpoint_url: + endpoint_url = "http://%s:%d/client/api"; % (cls.mgtSvrDetails["mgtSvrIp"], cls.mgtSvrDetails["port"]) + cls.debug("Setting endpoint.url to %s" % endpoint_url) + Configurations.update(cls.apiclient, "endpoint.url", endpoint_url) + + cls.initial_configuration_cks_enabled = Configurations.list( + cls.apiclient, name="cloud.kubernetes.service.enabled")[0].value + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Enabling CloudStack Kubernetes Service plugin and restarting management server") + Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "true") + cls.restartServer() + + cls.cks_service_offering = None + + if cls.setup_failed == False: + try: + cls.kubernetes_version = cls.addKubernetesSupportedVersion( + cls.services["cks_kubernetes_versions"][cls.k8s_version]) + cls.kubernetes_version_ids.append(cls.kubernetes_version.id) + except Exception as e: + cls.setup_failed = True + cls.debug("Failed to get Kubernetes version ISO in ready state: %s" % e) + + if cls.setup_failed == False: + cks_offering_data = cls.services["cks_service_offering"] + cks_offering_data["name"] = 'CKS-Instance-' + random_gen() + cls.cks_service_offering = ServiceOffering.create( + cls.apiclient, + cks_offering_data + ) + cls._cleanup.append(cls.cks_service_offering) + + cls.domain = get_domain(cls.apiclient) + cls.account = Account.create( + cls.apiclient, + cls.services["account"], + domainid=cls.domain.id + ) + cls._cleanup.append(cls.account) + + cls.default_network = None + + return + + @classmethod + def tearDownClass(cls): + # Delete added Kubernetes supported version + for version_id in cls.kubernetes_version_ids: + try: + cls.deleteKubernetesSupportedVersion(version_id) + except Exception as e: + cls.debug("Error during cleanup for Kubernetes versions: %s" % e) + + try: + # Restore CKS enabled + if cls.initial_configuration_cks_enabled not in ["true", True]: + cls.debug("Restoring Kubernetes Service enabled value") + Configurations.update(cls.apiclient, "cloud.kubernetes.service.enabled", "false") + cls.restartServer() + + cleanup_resources(cls.apiclient, cls._cleanup) + except Exception as e: + raise Exception("Warning: Exception during cleanup : %s" % e) Review Comment: Can try to incorporate `super(TestKubernetesClusterAffinityGroups, cls).tearDownClass()` here? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
