This is an automated email from the ASF dual-hosted git repository.
pearl11594 pushed a commit to branch nsx-k8s-cleanup
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit d7b877aeba9ed2f7f7ad7b097f8e2c0be66e50a5
Author: Pearl Dsilva <pearl1...@gmail.com>
AuthorDate: Wed Jan 17 13:01:34 2024 -0500
NSX: Cleanup NSX resources during k8s cluster cleanup
---
.../cluster/KubernetesClusterHelper.java | 1 +
.../cluster/KubernetesClusterHelperImpl.java | 13 ++++++++++
.../cluster/dao/KubernetesClusterVmMapDao.java | 2 ++
.../cluster/dao/KubernetesClusterVmMapDaoImpl.java | 12 ++++++++++
.../org/apache/cloudstack/service/NsxElement.java | 21 +++++++++-------
.../main/java/com/cloud/vm/UserVmManagerImpl.java | 28 ++++++++++++++++++----
.../core/spring-server-core-managers-context.xml | 5 ++--
7 files changed, 66 insertions(+), 16 deletions(-)
diff --git
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
index e445e50f82c..e160227749d 100644
---
a/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
+++
b/api/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelper.java
@@ -22,4 +22,5 @@ import org.apache.cloudstack.acl.ControlledEntity;
public interface KubernetesClusterHelper extends Adapter {
ControlledEntity findByUuid(String uuid);
+ ControlledEntity findByVmId(long vmId);
}
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
index 0ef916ab959..60bd81c7c5a 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterHelperImpl.java
@@ -17,6 +17,7 @@
package com.cloud.kubernetes.cluster;
import com.cloud.kubernetes.cluster.dao.KubernetesClusterDao;
+import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao;
import com.cloud.utils.component.AdapterBase;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.framework.config.ConfigKey;
@@ -24,18 +25,30 @@ import org.apache.cloudstack.framework.config.Configurable;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+import java.util.Objects;
@Component
public class KubernetesClusterHelperImpl extends AdapterBase implements
KubernetesClusterHelper, Configurable {
@Inject
private KubernetesClusterDao kubernetesClusterDao;
+ @Inject
+ private KubernetesClusterVmMapDao kubernetesClusterVmMapDao;
@Override
public ControlledEntity findByUuid(String uuid) {
return kubernetesClusterDao.findByUuid(uuid);
}
+ @Override
+ public ControlledEntity findByVmId(long vmId) {
+ KubernetesClusterVmMapVO clusterVmMapVO =
kubernetesClusterVmMapDao.getClusterMapFromVmId(vmId);
+ if (Objects.isNull(clusterVmMapVO)) {
+ return null;
+ }
+ return kubernetesClusterDao.findById(clusterVmMapVO.getClusterId());
+ }
+
@Override
public String getConfigComponentName() {
return KubernetesClusterHelper.class.getSimpleName();
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
index 688a611ac99..45c0b79485c 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDao.java
@@ -23,6 +23,8 @@ import java.util.List;
public interface KubernetesClusterVmMapDao extends
GenericDao<KubernetesClusterVmMapVO, Long> {
public List<KubernetesClusterVmMapVO> listByClusterId(long clusterId);
+
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId);
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long
clusterId, List<Long> vmIds);
int removeByClusterIdAndVmIdsIn(long clusterId, List<Long> vmIds);
diff --git
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
index b9f2ec917b2..0d90a4cdaca 100644
---
a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
+++
b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/dao/KubernetesClusterVmMapDaoImpl.java
@@ -31,12 +31,17 @@ import com.cloud.utils.db.SearchCriteria;
public class KubernetesClusterVmMapDaoImpl extends
GenericDaoBase<KubernetesClusterVmMapVO, Long> implements
KubernetesClusterVmMapDao {
private final SearchBuilder<KubernetesClusterVmMapVO> clusterIdSearch;
+ private final SearchBuilder<KubernetesClusterVmMapVO> vmIdSearch;
public KubernetesClusterVmMapDaoImpl() {
clusterIdSearch = createSearchBuilder();
clusterIdSearch.and("clusterId",
clusterIdSearch.entity().getClusterId(), SearchCriteria.Op.EQ);
clusterIdSearch.and("vmIdsIN", clusterIdSearch.entity().getVmId(),
SearchCriteria.Op.IN);
clusterIdSearch.done();
+
+ vmIdSearch = createSearchBuilder();
+ vmIdSearch.and("vmId", vmIdSearch.entity().getVmId(),
SearchCriteria.Op.EQ);
+ vmIdSearch.done();
}
@Override
@@ -47,6 +52,13 @@ public class KubernetesClusterVmMapDaoImpl extends
GenericDaoBase<KubernetesClus
return listBy(sc, filter);
}
+ @Override
+ public KubernetesClusterVmMapVO getClusterMapFromVmId(long vmId) {
+ SearchCriteria<KubernetesClusterVmMapVO> sc = vmIdSearch.create();
+ sc.setParameters("vmId", vmId);
+ return findOneBy(sc);
+ }
+
@Override
public List<KubernetesClusterVmMapVO> listByClusterIdAndVmIdsIn(long
clusterId, List<Long> vmIds) {
SearchCriteria<KubernetesClusterVmMapVO> sc = clusterIdSearch.create();
diff --git
a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
index 0986c367cf9..571e68496d6 100644
---
a/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
+++
b/plugins/network-elements/nsx/src/main/java/org/apache/cloudstack/service/NsxElement.java
@@ -505,10 +505,12 @@ public class NsxElement extends AdapterBase implements
DhcpServiceProvider, Dns
if (!canHandle(network, Network.Service.PortForwarding)) {
return false;
}
+ boolean result = true;
for (PortForwardingRule rule : rules) {
IPAddressVO publicIp =
ApiDBUtils.findIpAddressById(rule.getSourceIpAddressId());
UserVm vm = ApiDBUtils.findUserVmById(rule.getVirtualMachineId());
- if (vm == null || networkModel.getNicInNetwork(vm.getId(),
network.getId()) == null) {
+ if ((vm == null && (rule.getState() != FirewallRule.State.Revoke))
||
+ (vm != null && networkModel.getNicInNetwork(vm.getId(),
network.getId()) == null)) {
continue;
}
NsxOpObject nsxObject = getNsxOpObject(network);
@@ -523,8 +525,8 @@ public class NsxElement extends AdapterBase implements
DhcpServiceProvider, Dns
.setNetworkResourceId(nsxObject.getNetworkResourceId())
.setNetworkResourceName(nsxObject.getNetworkResourceName())
.setVpcResource(nsxObject.isVpcResource())
- .setVmId(vm.getId())
- .setVmIp(vm.getPrivateIpAddress())
+ .setVmId(Objects.nonNull(vm) ? vm.getId() : 0)
+ .setVmIp(Objects.nonNull(vm) ? vm.getPrivateIpAddress() :
null)
.setPublicIp(publicIp.getAddress().addr())
.setPrivatePort(privatePort)
.setPublicPort(publicPort)
@@ -532,12 +534,12 @@ public class NsxElement extends AdapterBase implements
DhcpServiceProvider, Dns
.setProtocol(rule.getProtocol().toUpperCase(Locale.ROOT))
.build();
if (rule.getState() == FirewallRule.State.Add) {
- return nsxService.createPortForwardRule(networkRule);
+ result &= nsxService.createPortForwardRule(networkRule);
} else if (rule.getState() == FirewallRule.State.Revoke) {
- return nsxService.deletePortForwardRule(networkRule);
+ result &= nsxService.deletePortForwardRule(networkRule);
}
}
- return true;
+ return result;
}
public Pair<VpcVO, NetworkVO> getVpcOrNetwork(Long vpcId, long networkId) {
@@ -613,6 +615,7 @@ public class NsxElement extends AdapterBase implements
DhcpServiceProvider, Dns
@Override
public boolean applyLBRules(Network network, List<LoadBalancingRule>
rules) throws ResourceUnavailableException {
+ boolean result = true;
for (LoadBalancingRule loadBalancingRule : rules) {
if (loadBalancingRule.getState() == FirewallRule.State.Active) {
continue;
@@ -638,12 +641,12 @@ public class NsxElement extends AdapterBase implements
DhcpServiceProvider, Dns
.setAlgorithm(loadBalancingRule.getAlgorithm())
.build();
if (loadBalancingRule.getState() == FirewallRule.State.Add) {
- return nsxService.createLbRule(networkRule);
+ result &= nsxService.createLbRule(networkRule);
} else if (loadBalancingRule.getState() ==
FirewallRule.State.Revoke) {
- return nsxService.deleteLbRule(networkRule);
+ result &= nsxService.deleteLbRule(networkRule);
}
}
- return true;
+ return result;
}
@Override
diff --git a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
index 2927db638ab..844e0380757 100644
--- a/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
+++ b/server/src/main/java/com/cloud/vm/UserVmManagerImpl.java
@@ -51,6 +51,9 @@ import javax.naming.ConfigurationException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.ParserConfigurationException;
+import com.cloud.kubernetes.cluster.KubernetesClusterHelper;
+import com.cloud.network.dao.NsxProviderDao;
+import com.cloud.network.element.NsxProviderVO;
import org.apache.cloudstack.acl.ControlledEntity;
import org.apache.cloudstack.acl.ControlledEntity.ACLType;
import org.apache.cloudstack.acl.SecurityChecker.AccessType;
@@ -589,6 +592,8 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
@Inject
VMScheduleManager vmScheduleManager;
+ @Inject
+ NsxProviderDao nsxProviderDao;
private ScheduledExecutorService _executor = null;
private ScheduledExecutorService _vmIpFetchExecutor = null;
@@ -597,6 +602,7 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
private boolean _dailyOrHourly = false;
private int capacityReleaseInterval;
private ExecutorService _vmIpFetchThreadExecutor;
+ private List<KubernetesClusterHelper> kubernetesClusterHelpers;
private String _instance;
@@ -610,6 +616,14 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
private static final int NUM_OF_2K_BLOCKS = 512;
private static final int MAX_HTTP_POST_LENGTH = NUM_OF_2K_BLOCKS *
MAX_USER_DATA_LENGTH_BYTES;
+ public List<KubernetesClusterHelper> getKubernetesClusterHelpers() {
+ return kubernetesClusterHelpers;
+ }
+
+ public void setKubernetesClusterHelpers(final
List<KubernetesClusterHelper> kubernetesClusterHelpers) {
+ this.kubernetesClusterHelpers = kubernetesClusterHelpers;
+ }
+
@Inject
private OrchestrationService _orchSrvc;
@@ -2528,11 +2542,15 @@ public class UserVmManagerImpl extends ManagerBase
implements UserVmManager, Vir
}
// cleanup port forwarding rules
- if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
- s_logger.debug("Port forwarding rules are removed successfully as
a part of vm id=" + vmId + " expunge");
- } else {
- success = false;
- s_logger.warn("Fail to remove port forwarding rules as a part of
vm id=" + vmId + " expunge");
+ VMInstanceVO vmInstanceVO = _vmInstanceDao.findById(vmId);
+ NsxProviderVO nsx =
nsxProviderDao.findByZoneId(vmInstanceVO.getDataCenterId());
+ if (Objects.isNull(nsx) ||
Objects.isNull(kubernetesClusterHelpers.get(0).findByVmId(vmId))) {
+ if (_rulesMgr.revokePortForwardingRulesForVm(vmId)) {
+ s_logger.debug("Port forwarding rules are removed successfully
as a part of vm id=" + vmId + " expunge");
+ } else {
+ success = false;
+ s_logger.warn("Fail to remove port forwarding rules as a part
of vm id=" + vmId + " expunge");
+ }
}
// cleanup load balancer rules
diff --git
a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
index 7227264e229..9656d544fbb 100644
---
a/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
+++
b/server/src/main/resources/META-INF/cloudstack/core/spring-server-core-managers-context.xml
@@ -106,8 +106,9 @@
<bean id="configurationServerImpl"
class="com.cloud.server.ConfigurationServerImpl" />
-
- <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl" />
+ <bean id="userVmManagerImpl" class="com.cloud.vm.UserVmManagerImpl">
+ <property name="kubernetesClusterHelpers"
value="#{kubernetesClusterHelperRegistry.registered}" />
+ </bean>
<bean id="consoleProxyManagerImpl"
class="com.cloud.consoleproxy.ConsoleProxyManagerImpl">
<property name="consoleProxyAllocators"
