This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch 4.20
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit 19d6b979af64555955f20ad027876f5a139cea0a
Author: Abhishek Kumar <abhishek.mr...@gmail.com>
AuthorDate: Fri Feb 21 11:16:22 2025 +0530
cks: create separate service account in project
A separate service account will be created and added in the project, if
not exist already, when a Kubernetes cluster is deployed in a project.
This account will have a role with limited API access.
Cleanup clusters on owner account cleanup, delete service account
if needed
When the owner account of k8s clusters is deleted, while its node VMs
get expunged, the cluster entry in DB remain present. This fixes the
issue by cleaning up all clusters for the account deleted.
Project k8s service account will be deleted on account cleanup or when
there is no active k8s cluster remaining
Signed-off-by: Abhishek Kumar <abhishek.mr...@gmail.com>
---
.../cluster/KubernetesServiceHelper.java | 2 +
.../main/java/com/cloud/user/dao/AccountDao.java | 8 +-
.../java/com/cloud/user/dao/AccountDaoImpl.java | 25 +-
.../cluster/KubernetesClusterManagerImpl.java | 256 ++++++++++++++++++---
.../cluster/KubernetesClusterService.java | 8 +-
.../cluster/KubernetesServiceHelperImpl.java | 8 +
.../cluster/dao/KubernetesClusterDao.java | 4 +-
.../cluster/dao/KubernetesClusterDaoImpl.java | 33 ++-
.../java/com/cloud/user/AccountManagerImpl.java | 13 ++
9 files changed, 306 insertions(+), 51 deletions(-)
diff --git
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
index a13c1b3a6a8..84e898528b5 100644
---
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
+++
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelper.java
@@ -18,6 +18,7 @@ package com.cloud.kubernetes.cluster;
import org.apache.cloudstack.acl.ControlledEntity;
+import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.Adapter;
@@ -26,4 +27,5 @@ public interface KubernetesServiceHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
ControlledEntity findByVmId(long vmId);
void checkVmCanBeDestroyed(UserVm userVm);
+ void cleanupForAccount(Account account);
}
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
index 17b07496731..dae5f3a3467 100644
--- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
+++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDao.java
@@ -16,6 +16,9 @@
// under the License.
package com.cloud.user.dao;
+import java.util.Date;
+import java.util.List;
+
import com.cloud.user.Account;
import com.cloud.user.AccountVO;
import com.cloud.user.User;
@@ -23,9 +26,6 @@ import com.cloud.utils.Pair;
import com.cloud.utils.db.Filter;
import com.cloud.utils.db.GenericDao;
-import java.util.Date;
-import java.util.List;
-
public interface AccountDao extends GenericDao<AccountVO, Long> {
Pair<User, Account> findUserAccountByApiKey(String apiKey);
@@ -33,6 +33,8 @@ public interface AccountDao extends GenericDao<AccountVO,
Long> {
Pair<List<AccountVO>, Integer> findAccountsLike(String accountName, Filter
filter);
+ List<AccountVO> findAccountsByName(String accountName);
+
List<AccountVO> findActiveAccounts(Long maxAccountId, Filter filter);
List<AccountVO> findRecentlyDeletedAccounts(Long maxAccountId, Date
earliestRemovedDate, Filter filter);
diff --git a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
index 2654b22374f..f5f95d5da1f 100644
--- a/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/user/dao/AccountDaoImpl.java
@@ -16,6 +16,14 @@
// under the License.
package com.cloud.user.dao;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.stereotype.Component;
+
import com.cloud.user.Account;
import com.cloud.user.Account.State;
import com.cloud.user.AccountVO;
@@ -30,14 +38,7 @@ import com.cloud.utils.db.SearchBuilder;
import com.cloud.utils.db.SearchCriteria;
import com.cloud.utils.db.SearchCriteria.Func;
import com.cloud.utils.db.SearchCriteria.Op;
-import org.apache.commons.lang3.StringUtils;
import com.cloud.utils.db.TransactionLegacy;
-import org.springframework.stereotype.Component;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.util.Date;
-import java.util.List;
@Component
public class AccountDaoImpl extends GenericDaoBase<AccountVO, Long> implements
AccountDao {
@@ -190,6 +191,16 @@ public class AccountDaoImpl extends
GenericDaoBase<AccountVO, Long> implements A
return searchAndCount(sc, filter);
}
+ @Override
+ public List<AccountVO> findAccountsByName(String accountName) {
+ SearchBuilder<AccountVO> sb = createSearchBuilder();
+ sb.and("accountName", sb.entity().getAccountName(),
SearchCriteria.Op.EQ);
+ sb.done();
+ SearchCriteria<AccountVO> sc = sb.create();
+ sc.setParameters("accountName", accountName);
+ return search(sc, null);
+ }
+
@Override
public Account findEnabledAccount(String accountName, Long domainId) {
SearchCriteria<AccountVO> sc = AllFieldsSearch.create("accountName",
accountName);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
index 131d7b22606..5e86bb6daf0 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
@@ -44,6 +44,11 @@ import javax.naming.ConfigurationException;
import com.cloud.uservm.UserVm;
import com.cloud.vm.UserVmService;
import org.apache.cloudstack.acl.ControlledEntity;
+import org.apache.cloudstack.acl.Role;
+import org.apache.cloudstack.acl.RolePermissionEntity;
+import org.apache.cloudstack.acl.RoleService;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.acl.Rule;
import org.apache.cloudstack.acl.SecurityChecker;
import org.apache.cloudstack.annotation.AnnotationService;
import org.apache.cloudstack.annotation.dao.AnnotationDao;
@@ -52,6 +57,14 @@ import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.api.ApiConstants.VMDetails;
import org.apache.cloudstack.api.BaseCmd;
import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.address.AssociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.DisassociateIPAddrCmd;
+import org.apache.cloudstack.api.command.user.address.ListPublicIpAddressesCmd;
+import org.apache.cloudstack.api.command.user.firewall.CreateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.DeleteFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.firewall.ListFirewallRulesCmd;
+import org.apache.cloudstack.api.command.user.firewall.UpdateFirewallRuleCmd;
+import org.apache.cloudstack.api.command.user.job.QueryAsyncJobResultCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -62,6 +75,18 @@ import
org.apache.cloudstack.api.command.user.kubernetes.cluster.ScaleKubernetes
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.StartKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.StopKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.UpgradeKubernetesClusterCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.AssignToLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.CreateLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.DeleteLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRuleInstancesCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.ListLoadBalancerRulesCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.RemoveFromLoadBalancerRuleCmd;
+import
org.apache.cloudstack.api.command.user.loadbalancer.UpdateLoadBalancerRuleCmd;
+import org.apache.cloudstack.api.command.user.network.CreateNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.DeleteNetworkACLCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworkACLsCmd;
+import org.apache.cloudstack.api.command.user.network.ListNetworksCmd;
+import org.apache.cloudstack.api.command.user.vm.ListVMsCmd;
import org.apache.cloudstack.api.response.KubernetesClusterConfigResponse;
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.ListResponse;
@@ -147,6 +172,8 @@ import com.cloud.offerings.dao.NetworkOfferingServiceMapDao;
import com.cloud.org.Cluster;
import com.cloud.org.Grouping;
import com.cloud.projects.Project;
+import com.cloud.projects.ProjectAccount;
+import com.cloud.projects.ProjectManager;
import com.cloud.resource.ResourceManager;
import com.cloud.service.ServiceOfferingVO;
import com.cloud.service.dao.ServiceOfferingDao;
@@ -155,14 +182,17 @@ import com.cloud.storage.dao.VMTemplateDao;
import com.cloud.user.Account;
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
+import com.cloud.user.AccountVO;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.User;
import com.cloud.user.UserAccount;
import com.cloud.user.UserVO;
+import com.cloud.user.dao.AccountDao;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
+import com.cloud.utils.UuidUtils;
import com.cloud.utils.component.ComponentContext;
import com.cloud.utils.component.ManagerBase;
import com.cloud.utils.concurrency.NamedThreadFactory;
@@ -186,12 +216,40 @@ import org.apache.logging.log4j.Level;
public class KubernetesClusterManagerImpl extends ManagerBase implements
KubernetesClusterService {
private static final String
DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME =
"DefaultNetworkOfferingforKubernetesService";
+
private static final String
DEFAULT_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network
Offering used for CloudStack Kubernetes service";
private static final String
DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME =
"DefaultNSXNetworkOfferingforKubernetesService";
private static final String
DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME =
"DefaultNSXVPCNetworkOfferingforKubernetesService";
private static final String
DEFAULT_NSX_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network
Offering for NSX CloudStack Kubernetes Service";
private static final String
DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT = "Network
Offering for NSX CloudStack Kubernetes service on VPC";
+ private static final List<Class<?>>
PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS = Arrays.asList(
+ QueryAsyncJobResultCmd.class,
+ ListVMsCmd.class,
+ ListNetworksCmd.class,
+ ListPublicIpAddressesCmd.class,
+ AssociateIPAddrCmd.class,
+ DisassociateIPAddrCmd.class,
+ ListLoadBalancerRulesCmd.class,
+ CreateLoadBalancerRuleCmd.class,
+ UpdateLoadBalancerRuleCmd.class,
+ DeleteLoadBalancerRuleCmd.class,
+ AssignToLoadBalancerRuleCmd.class,
+ RemoveFromLoadBalancerRuleCmd.class,
+ ListLoadBalancerRuleInstancesCmd.class,
+ ListFirewallRulesCmd.class,
+ CreateFirewallRuleCmd.class,
+ UpdateFirewallRuleCmd.class,
+ DeleteFirewallRuleCmd.class,
+ ListNetworkACLsCmd.class,
+ CreateNetworkACLCmd.class,
+ DeleteNetworkACLCmd.class,
+ ListKubernetesClustersCmd.class,
+ ScaleKubernetesClusterCmd.class
+ );
+ private static final String PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME =
"Kubernetes";
+ private static final String PROJECT_KUBERNETES_ACCOUNT_LAST_NAME =
"Service User";
+
protected StateMachine2<KubernetesCluster.State, KubernetesCluster.Event,
KubernetesCluster> _stateMachine = KubernetesCluster.State.getStateMachine();
ScheduledExecutorService _gcExecutor;
@@ -263,11 +321,16 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
public SecurityGroupService securityGroupService;
@Inject
public NetworkHelper networkHelper;
-
@Inject
private UserVmService userVmService;
@Inject
RoutedIpv4Manager routedIpv4Manager;
+ @Inject
+ public AccountDao accountDao;
+ @Inject
+ public ProjectManager projectManager;
+ @Inject
+ RoleService roleService;
private void logMessage(final Level logLevel, final String message, final
Exception e) {
if (logLevel == Level.WARN) {
@@ -1380,23 +1443,31 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
}
- private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster)
{
- Account owner =
accountService.getActiveAccountById(kubernetesCluster.getAccountId());
- if (owner == null || owner.getType() == Account.Type.PROJECT) {
- owner = CallContext.current().getCallingAccount();
+ protected String[] createUserApiKeyAndSecretKey(long userId) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ return accountService.createApiKeyAndSecretKey(userId);
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected String[] getServiceUserKeys(Account owner) {
+ String username = owner.getAccountName();
+ if (!username.startsWith(KUBEADMIN_ACCOUNT_NAME + "-")) {
+ username += "-" + KUBEADMIN_ACCOUNT_NAME;
}
- String username = owner.getAccountName() + "-" +
KUBEADMIN_ACCOUNT_NAME;
UserAccount kubeadmin = accountService.getActiveUserAccount(username,
owner.getDomainId());
- String[] keys = null;
+ String[] keys;
if (kubeadmin == null) {
User kube = userDao.persist(new UserVO(owner.getAccountId(),
username, UUID.randomUUID().toString(), owner.getAccountName(),
- KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null,
UUID.randomUUID().toString(), User.Source.UNKNOWN));
- keys = accountService.createApiKeyAndSecretKey(kube.getId());
+ KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null,
UUID.randomUUID().toString(), User.Source.UNKNOWN));
+ keys = createUserApiKeyAndSecretKey(kube.getId());
} else {
String apiKey = kubeadmin.getApiKey();
String secretKey = kubeadmin.getSecretKey();
if (StringUtils.isAnyEmpty(apiKey, secretKey)) {
- keys =
accountService.createApiKeyAndSecretKey(kubeadmin.getId());
+ keys = createUserApiKeyAndSecretKey(kubeadmin.getId());
} else {
keys = new String[]{apiKey, secretKey};
}
@@ -1404,6 +1475,76 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
return keys;
}
+ protected Role createProjectKubernetesAccountRole() {
+ Role role =
roleService.createRole(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, RoleType.User,
+ PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME, false);
+ for (Class<?> allowedApi :
PROJECT_KUBERNETES_ACCOUNT_ROLE_ALLOWED_APIS) {
+ final String apiName = BaseCmd.getCommandNameByClass(allowedApi);
+ roleService.createRolePermission(role, new Rule(apiName),
RolePermissionEntity.Permission.ALLOW,
+ String.format("Allow %s", apiName));
+ }
+ roleService.createRolePermission(role, new Rule("*"),
RolePermissionEntity.Permission.DENY,
+ "Deny all");
+ logger.debug(String.format("Created default role for Kubernetes
service account in projects: %s", role));
+ return role;
+ }
+
+ public Role getProjectKubernetesAccountRole() {
+ List<Role> roles =
roleService.findRolesByName(PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME);
+ if (CollectionUtils.isNotEmpty(roles)) {
+ Role role = roles.get(0);
+ logger.debug(String.format("Found default role for Kubernetes
service account in projects: %s", role));
+ return role;
+ }
+ return createProjectKubernetesAccountRole();
+ }
+
+ protected Account createProjectKubernetesAccount(final Project project,
final String accountName) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ Role role = getProjectKubernetesAccountRole();
+ UserAccount userAccount =
accountService.createUserAccount(accountName,
+ UuidUtils.first(UUID.randomUUID().toString()),
PROJECT_KUBERNETES_ACCOUNT_FIRST_NAME,
+ PROJECT_KUBERNETES_ACCOUNT_LAST_NAME, null, null,
accountName, Account.Type.NORMAL, role.getId(),
+ project.getDomainId(), null, null, null, null,
User.Source.NATIVE);
+ projectManager.assignAccountToProject(project,
userAccount.getAccountId(), ProjectAccount.Role.Regular,
+ userAccount.getId(), null);
+ Account account =
accountService.getAccount(userAccount.getAccountId());
+ logger.debug(String.format("Created Kubernetes service account in
project %s: %s", project, account));
+ return account;
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected Account getProjectKubernetesAccount(final Account callerAccount,
final boolean create) {
+ Project project =
ApiDBUtils.findProjectByProjectAccountId(callerAccount.getId());
+ final String accountName = String.format("%s-%s",
KUBEADMIN_ACCOUNT_NAME, UuidUtils.first(project.getUuid()));
+ List<AccountVO> accounts = accountDao.findAccountsByName(accountName);
+ for (AccountVO account : accounts) {
+ if (projectManager.canAccessProjectAccount(account,
project.getProjectAccountId())) {
+ logger.debug(String.format("Created Kubernetes service account
in project %s: %s", project, account));
+ return account;
+ }
+ }
+ return create ? createProjectKubernetesAccount(project, accountName) :
null;
+ }
+
+ protected Account getProjectKubernetesAccount(final Account callerAccount)
{
+ return getProjectKubernetesAccount(callerAccount, true);
+ }
+
+ private String[] getServiceUserKeys(KubernetesClusterVO kubernetesCluster)
{
+ Account owner =
accountService.getActiveAccountById(kubernetesCluster.getAccountId());
+ if (owner == null) {
+ owner = CallContext.current().getCallingAccount();
+ }
+ if (owner.getType() == Account.Type.PROJECT) {
+ owner = getProjectKubernetesAccount(owner);
+ }
+ return getServiceUserKeys(owner);
+ }
+
@Override
@ActionEvent(eventType =
KubernetesClusterEventTypes.EVENT_KUBERNETES_CLUSTER_STOP,
eventDescription = "stopping Kubernetes cluster", async = true)
@@ -1448,15 +1589,13 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
Long kubernetesClusterId = cmd.getId();
- KubernetesClusterVO cluster =
kubernetesClusterDao.findById(kubernetesClusterId);
+ final KubernetesClusterVO cluster =
kubernetesClusterDao.findById(kubernetesClusterId);
if (cluster == null) {
throw new InvalidParameterValueException("Invalid cluster id
specified");
}
accountManager.checkAccess(CallContext.current().getCallingAccount(),
SecurityChecker.AccessType.OperateEntry, false, cluster);
if (cluster.getClusterType() ==
KubernetesCluster.ClusterType.CloudManaged) {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(cluster, this);
- destroyWorker = ComponentContext.inject(destroyWorker);
- return destroyWorker.destroy();
+ return destroyKubernetesCluster(cluster);
} else {
boolean cleanup = cmd.getCleanup();
boolean expunge = cmd.getExpunge();
@@ -1483,13 +1622,14 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
}
}
- return Transaction.execute(new TransactionCallback<Boolean>() {
- @Override
- public Boolean doInTransaction(TransactionStatus status) {
-
kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
-
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
- return kubernetesClusterDao.remove(kubernetesClusterId);
+ return Transaction.execute((TransactionCallback<Boolean>) status
-> {
+ kubernetesClusterDetailsDao.removeDetails(kubernetesClusterId);
+
kubernetesClusterVmMapDao.removeByClusterId(kubernetesClusterId);
+ if (kubernetesClusterDao.remove(kubernetesClusterId)) {
+ deleteProjectKubernetesAccountIfNeeded(cluster);
+ return true;
}
+ return false;
});
}
}
@@ -1752,6 +1892,66 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
return responseList;
}
+ protected void deleteProjectKubernetesAccount(Account projectAccount) {
+ CallContext.register(User.UID_SYSTEM, Account.ACCOUNT_ID_SYSTEM);
+ try {
+ Account serviceAccount =
getProjectKubernetesAccount(projectAccount, false);
+ if (serviceAccount != null) {
+
accountManager.deleteAccount(accountDao.findById(serviceAccount.getId()),
User.UID_SYSTEM,
+ accountService.getSystemAccount());
+ }
+ } finally {
+ CallContext.unregister();
+ }
+ }
+
+ protected void deleteProjectKubernetesAccountIfNeeded(final
KubernetesCluster kubernetesCluster) {
+ Account owner =
accountService.getAccount(kubernetesCluster.getAccountId());
+ if (owner == null) {
+ return;
+ }
+ if (Account.Type.PROJECT.equals(owner.getType()) &&
+
kubernetesClusterDao.countNotForGCByAccount(owner.getAccountId()) == 0) {
+ deleteProjectKubernetesAccount(owner);
+ }
+ }
+
+ protected boolean destroyKubernetesCluster(KubernetesCluster
kubernetesCluster, boolean deleteProjectAccount) {
+ KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
+ KubernetesClusterManagerImpl.this);
+ destroyWorker = ComponentContext.inject(destroyWorker);
+ boolean result = destroyWorker.destroy();
+ if (deleteProjectAccount) {
+ deleteProjectKubernetesAccountIfNeeded(kubernetesCluster);
+ }
+ return result;
+ }
+
+ protected boolean destroyKubernetesCluster(KubernetesCluster
kubernetesCluster) {
+ return destroyKubernetesCluster(kubernetesCluster, true);
+ }
+
+ @Override
+ public void cleanupForAccount(Account account) {
+ List<KubernetesClusterVO> clusters =
kubernetesClusterDao.listForCleanupByAccount(account.getId());
+ if (CollectionUtils.isEmpty(clusters)) {
+ return;
+ }
+ logger.debug(String.format("Cleaning up %d Kubernetes cluster for %s",
clusters.size(), account));
+ for (KubernetesClusterVO cluster : clusters) {
+ try {
+ destroyKubernetesCluster(cluster, false);
+ } catch (CloudRuntimeException e) {
+ logger.warn(String.format("Failed to destroy Kubernetes
cluster: %s during cleanup for %s",
+ cluster.getName(), account), e);
+ }
+ }
+ if (!Account.Type.PROJECT.equals(account.getType())) {
+ return;
+ }
+ deleteProjectKubernetesAccount(account);
+ }
+
@Override
public List<Class<?>> getCommands() {
List<Class<?>> cmdList = new ArrayList<Class<?>>();
@@ -1803,12 +2003,8 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
logger.info("Running Kubernetes cluster garbage
collector on Kubernetes cluster: {}", kubernetesCluster);
}
try {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
KubernetesClusterManagerImpl.this);
- destroyWorker = ComponentContext.inject(destroyWorker);
- if (destroyWorker.destroy()) {
- if (logger.isInfoEnabled()) {
- logger.info("Garbage collection complete for
Kubernetes cluster: {}", kubernetesCluster);
- }
+ if (destroyKubernetesCluster(kubernetesCluster)) {
+ logger.info("Garbage collection complete for
Kubernetes cluster: {}", kubernetesCluster);
} else {
logger.warn("Garbage collection failed for
Kubernetes cluster : {}, it will be attempted to garbage collected in next
run", kubernetesCluster);
}
@@ -1931,9 +2127,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
logger.info("Running Kubernetes cluster state
scanner on Kubernetes cluster: {} for state: {}", kubernetesCluster,
KubernetesCluster.State.Destroying.toString());
}
try {
- KubernetesClusterDestroyWorker destroyWorker = new
KubernetesClusterDestroyWorker(kubernetesCluster,
KubernetesClusterManagerImpl.this);
- destroyWorker =
ComponentContext.inject(destroyWorker);
- destroyWorker.destroy();
+ destroyKubernetesCluster(kubernetesCluster);
} catch (Exception e) {
logger.warn("Failed to run Kubernetes cluster
Destroying state scanner on Kubernetes cluster : {} status scanner",
kubernetesCluster, e);
}
@@ -1947,7 +2141,7 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
}
// checks if Kubernetes cluster is in desired state
- boolean isClusterVMsInDesiredState(KubernetesCluster kubernetesCluster,
VirtualMachine.State state) {
+ private boolean isClusterVMsInDesiredState(KubernetesCluster
kubernetesCluster, VirtualMachine.State state) {
List<KubernetesClusterVmMapVO> clusterVMs =
kubernetesClusterVmMapDao.listByClusterId(kubernetesCluster.getId());
// check cluster is running at desired capacity include control nodes
as well
@@ -1985,6 +2179,8 @@ public class KubernetesClusterManagerImpl extends
ManagerBase implements Kuberne
createNetworkOfferingForKubernetes(DEFAULT_NSX_VPC_TIER_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_NAME,
DEFAULT_NSX_VPC_NETWORK_OFFERING_FOR_KUBERNETES_SERVICE_DISPLAY_TEXT , true,
true);
+ getProjectKubernetesAccountRole();
+
_gcExecutor.scheduleWithFixedDelay(new
KubernetesClusterGarbageCollector(), 300, 300, TimeUnit.SECONDS);
_stateScanner.scheduleWithFixedDelay(new
KubernetesClusterStatusScanner(), 300, 30, TimeUnit.SECONDS);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
index 9d86c564de4..0c2338465de 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
@@ -16,6 +16,8 @@
// under the License.
package com.cloud.kubernetes.cluster;
+import java.util.List;
+
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.AddVirtualMachinesToKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.CreateKubernetesClusterCmd;
import
org.apache.cloudstack.api.command.user.kubernetes.cluster.DeleteKubernetesClusterCmd;
@@ -34,16 +36,16 @@ import org.apache.cloudstack.framework.config.ConfigKey;
import org.apache.cloudstack.framework.config.Configurable;
import com.cloud.network.Network;
+import com.cloud.user.Account;
import com.cloud.utils.component.PluggableService;
import com.cloud.utils.exception.CloudRuntimeException;
-import java.util.List;
-
public interface KubernetesClusterService extends PluggableService,
Configurable {
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";
+ String PROJECT_KUBEADMIN_ACCOUNT_ROLE_NAME = "Project Kubernetes Service
Role";
static final ConfigKey<Boolean> KubernetesServiceEnabled = new
ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.service.enabled",
@@ -125,4 +127,6 @@ public interface KubernetesClusterService extends
PluggableService, Configurable
List<RemoveVirtualMachinesFromKubernetesClusterResponse>
removeVmsFromCluster(RemoveVirtualMachinesFromKubernetesClusterCmd cmd);
boolean isDirectAccess(Network network);
+
+ void cleanupForAccount(Account account);
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
index bf49c2abb8d..d7e6f65ca05 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesServiceHelperImpl.java
@@ -32,6 +32,7 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.kubernetes.version.KubernetesSupportedVersion;
import com.cloud.kubernetes.version.KubernetesVersionEventTypes;
+import com.cloud.user.Account;
import com.cloud.uservm.UserVm;
import com.cloud.utils.component.AdapterBase;
import com.cloud.utils.exception.CloudRuntimeException;
@@ -50,6 +51,8 @@ public class KubernetesServiceHelperImpl extends AdapterBase
implements Kubernet
private KubernetesClusterDao kubernetesClusterDao;
@Inject
private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
+ @Inject
+ KubernetesClusterService kubernetesClusterService;
protected void setEventTypeEntityDetails(Class<?> eventTypeDefinedClass,
Class<?> entityClass) {
Field[] declaredFields = eventTypeDefinedClass.getDeclaredFields();
@@ -106,6 +109,11 @@ public class KubernetesServiceHelperImpl extends
AdapterBase implements Kubernet
throw new CloudRuntimeException(msg);
}
+ @Override
+ public void cleanupForAccount(Account account) {
+ kubernetesClusterService.cleanupForAccount(account);
+ }
+
@Override
public String getConfigComponentName() {
return KubernetesServiceHelper.class.getSimpleName();
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
index 9341912012f..7df6a6b1dce 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDao.java
@@ -25,8 +25,8 @@ import com.cloud.utils.fsm.StateDao;
public interface KubernetesClusterDao extends GenericDao<KubernetesClusterVO,
Long>,
StateDao<KubernetesCluster.State, KubernetesCluster.Event,
KubernetesCluster> {
-
- List<KubernetesClusterVO> listByAccount(long accountId);
+ List<KubernetesClusterVO> listForCleanupByAccount(long accountId);
+ int countNotForGCByAccount(long accountId);
List<KubernetesClusterVO> findKubernetesClustersToGarbageCollect();
List<KubernetesClusterVO>
findManagedKubernetesClustersInState(KubernetesCluster.State state);
List<KubernetesClusterVO> listByNetworkId(long networkId);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
index 63cca3563f7..7bec98d5d25 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterDaoImpl.java
@@ -30,16 +30,25 @@ import com.cloud.utils.db.TransactionLegacy;
@Component
public class KubernetesClusterDaoImpl extends
GenericDaoBase<KubernetesClusterVO, Long> implements KubernetesClusterDao {
- private final SearchBuilder<KubernetesClusterVO> AccountIdSearch;
+ private final SearchBuilder<KubernetesClusterVO> CleanupAccountIdSearch;
+ private final SearchBuilder<KubernetesClusterVO> NotForGCByAccountIDCount;
private final SearchBuilder<KubernetesClusterVO> GarbageCollectedSearch;
private final SearchBuilder<KubernetesClusterVO> ManagedStateSearch;
private final SearchBuilder<KubernetesClusterVO> SameNetworkSearch;
private final SearchBuilder<KubernetesClusterVO> KubernetesVersionSearch;
public KubernetesClusterDaoImpl() {
- AccountIdSearch = createSearchBuilder();
- AccountIdSearch.and("account",
AccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
- AccountIdSearch.done();
+ CleanupAccountIdSearch = createSearchBuilder();
+ CleanupAccountIdSearch.and("account",
CleanupAccountIdSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
+ CleanupAccountIdSearch.and("cluster_type",
CleanupAccountIdSearch.entity().getClusterType(), SearchCriteria.Op.EQ);
+ CleanupAccountIdSearch.done();
+
+ NotForGCByAccountIDCount = createSearchBuilder();
+ NotForGCByAccountIDCount.and("gc",
NotForGCByAccountIDCount.entity().isCheckForGc(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.and("account",
NotForGCByAccountIDCount.entity().getAccountId(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.and("cluster_type",
NotForGCByAccountIDCount.entity().getClusterType(), SearchCriteria.Op.EQ);
+ NotForGCByAccountIDCount.select(null, SearchCriteria.Func.COUNT, null);
+ NotForGCByAccountIDCount.done();
GarbageCollectedSearch = createSearchBuilder();
GarbageCollectedSearch.and("gc",
GarbageCollectedSearch.entity().isCheckForGc(), SearchCriteria.Op.EQ);
@@ -62,10 +71,20 @@ public class KubernetesClusterDaoImpl extends
GenericDaoBase<KubernetesClusterVO
}
@Override
- public List<KubernetesClusterVO> listByAccount(long accountId) {
- SearchCriteria<KubernetesClusterVO> sc = AccountIdSearch.create();
+ public List<KubernetesClusterVO> listForCleanupByAccount(long accountId) {
+ SearchCriteria<KubernetesClusterVO> sc =
CleanupAccountIdSearch.create();
+ sc.setParameters("cluster_type",
KubernetesCluster.ClusterType.CloudManaged);
+ sc.setParameters("account", accountId);
+ return listBy(sc);
+ }
+
+ @Override
+ public int countNotForGCByAccount(long accountId) {
+ SearchCriteria<KubernetesClusterVO> sc =
NotForGCByAccountIDCount.create();
+ sc.setParameters("cluster_type",
KubernetesCluster.ClusterType.CloudManaged);
sc.setParameters("account", accountId);
- return listBy(sc, null);
+ sc.setParameters("gc", false);
+ return getCount(sc);
}
@Override
diff --git a/server/src/main/java/com/cloud/user/AccountManagerImpl.java
b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
index db2b5f32a7f..177f821d178 100644
--- a/server/src/main/java/com/cloud/user/AccountManagerImpl.java
+++ b/server/src/main/java/com/cloud/user/AccountManagerImpl.java
@@ -120,6 +120,7 @@ import com.cloud.exception.OperationTimedoutException;
import com.cloud.exception.PermissionDeniedException;
import com.cloud.exception.ResourceUnavailableException;
import com.cloud.host.dao.HostDao;
+import com.cloud.kubernetes.cluster.KubernetesServiceHelper;
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
@@ -887,6 +888,16 @@ public class AccountManagerImpl extends ManagerBase
implements AccountManager, M
return cleanupAccount(account, callerUserId, caller);
}
+ protected void cleanupPluginsResourcesIfNeeded(Account account) {
+ try {
+ KubernetesServiceHelper kubernetesServiceHelper =
+
ComponentContext.getDelegateComponentOfType(KubernetesServiceHelper.class);
+ kubernetesServiceHelper.cleanupForAccount(account);
+ } catch (NoSuchBeanDefinitionException ignored) {
+ logger.debug("No KubernetesServiceHelper bean found");
+ }
+ }
+
protected boolean cleanupAccount(AccountVO account, long callerUserId,
Account caller) {
long accountId = account.getId();
boolean accountCleanupNeeded = false;
@@ -968,6 +979,8 @@ public class AccountManagerImpl extends ManagerBase
implements AccountManager, M
}
}
+ cleanupPluginsResourcesIfNeeded(account);
+
// Destroy the account's VMs
List<UserVmVO> vms = _userVmDao.listByAccountId(accountId);
if (logger.isDebugEnabled()) {
