This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/airavata-mft.git
commit 1bc3594d1c63b07fbcd4f50f9bb297e9aa2b2d5d Author: Dimuthu Wannipurage <[email protected]> AuthorDate: Thu Feb 2 13:21:45 2023 -0500 Ignoring failed Agent instances when reusing --- .../mft/controller/spawner/AgentOrchestrator.java | 8 +-- .../mft/controller/spawner/AgentSpawner.java | 2 +- .../mft/controller/spawner/EC2AgentSpawner.java | 63 +++++++++++++++------- 3 files changed, 49 insertions(+), 24 deletions(-) diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentOrchestrator.java b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentOrchestrator.java index cb58002..ceceab6 100644 --- a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentOrchestrator.java +++ b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentOrchestrator.java @@ -32,7 +32,7 @@ public class AgentOrchestrator { private static final Logger logger = LoggerFactory.getLogger(AgentOrchestrator.class); - private final int SPAWNER_MAX_IDLE_SECONDS = 30; + private final int SPAWNER_MAX_IDLE_SECONDS = 30; // TODO Externalize this private class TransferInfo { private final String transferId; @@ -145,7 +145,7 @@ public class AgentOrchestrator { logger.info("Removing consul key {}", transferInfo.consulKey); transferDispatcher.getMftConsulClient().getKvClient().deleteKey(transferInfo.consulKey); logger.info("Terminating the spawner"); - metadata.spawner.terminate(); + metadata.spawner.terminate(true); launchedSpawnersMap.remove(key); } finally { metadata.transferInfos.remove(agentTransferId); @@ -169,7 +169,7 @@ public class AgentOrchestrator { } catch (Exception e) { logger.info("Killing spawner with key {} as the agent is not responding and inactive for {} seconds", key, SPAWNER_MAX_IDLE_SECONDS); - metadata.spawner.terminate(); + metadata.spawner.terminate(false); launchedSpawnersMap.remove(key); return; } @@ -181,7 +181,7 @@ public class AgentOrchestrator { logger.info("Killing spawner with key {} as all files were transferred and the agent" + " is inactive for {} seconds", key, SPAWNER_MAX_IDLE_SECONDS); - metadata.spawner.terminate(); + metadata.spawner.terminate(false); launchedSpawnersMap.remove(key); } } diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentSpawner.java b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentSpawner.java index a8a733a..3dadf4d 100644 --- a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentSpawner.java +++ b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/AgentSpawner.java @@ -33,7 +33,7 @@ public abstract class AgentSpawner { public abstract void launch(); public abstract Future<String> getLaunchState(); - public abstract void terminate(); + public abstract void terminate(boolean failed); public abstract Future<Boolean> getTerminateState(); } diff --git a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/EC2AgentSpawner.java b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/EC2AgentSpawner.java index 0ac8094..30cf81e 100644 --- a/controller/src/main/java/org/apache/airavata/mft/controller/spawner/EC2AgentSpawner.java +++ b/controller/src/main/java/org/apache/airavata/mft/controller/spawner/EC2AgentSpawner.java @@ -23,6 +23,7 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.ec2.AmazonEC2; import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; +import com.amazonaws.services.lightsail.model.CreateInstanceSnapshotRequest; import org.apache.airavata.mft.agent.stub.SecretWrapper; import org.apache.airavata.mft.agent.stub.StorageWrapper; import org.slf4j.Logger; @@ -109,6 +110,20 @@ public class EC2AgentSpawner extends AgentSpawner { .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); + boolean preSavedImage = false; + DescribeImagesRequest describeImagesRequest = new DescribeImagesRequest().withFilters( + new ArrayList<>()); + describeImagesRequest.getFilters().add(new Filter().withName("name").withValues("mft-agent")); + DescribeImagesResult describeImagesResult = amazonEC2.describeImages( + describeImagesRequest); + List<Image> images = describeImagesResult.getImages(); + if (!images.isEmpty()) { + Image image = images.get(0); + imageId = image.getImageId(); + preSavedImage = true; + logger.info("Using already created image {}", imageId); + } + DescribeSecurityGroupsRequest desSecGrp = new DescribeSecurityGroupsRequest(); DescribeSecurityGroupsResult describeSecurityGroupsResult = amazonEC2.describeSecurityGroups(desSecGrp); List<SecurityGroup> securityGroups = describeSecurityGroupsResult.getSecurityGroups(); @@ -178,14 +193,15 @@ public class EC2AgentSpawner extends AgentSpawner { RunInstancesRequest runInstancesRequest = new RunInstancesRequest(); runInstancesRequest.withImageId(imageId) - .withInstanceType(InstanceType.T1Micro) + .withInstanceType(InstanceType.T1Micro) // TODO Externalize .withMinCount(1) .withMaxCount(1) .withKeyName(keyName) .withTagSpecifications( new TagSpecification().withResourceType(ResourceType.Instance) .withTags(new Tag().withKey("Type").withValue("MFT-Agent"), - new Tag().withKey("AgentId").withValue(agentId))) + new Tag().withKey("AgentId").withValue(agentId), + new Tag().withKey("Name").withValue("MFT-Agent"))) .withSecurityGroups(secGroupName); @@ -225,23 +241,25 @@ public class EC2AgentSpawner extends AgentSpawner { Path.of(mftKeyDir, keyName).toAbsolutePath().toString(), systemUser); logger.info("Created SSH Connection. Installing dependencies..."); - int exeCode = sshProvider.runCommand("sudo apt update -y"); - if (exeCode != 0) - throw new IOException("Failed to update apt for VM"); - exeCode = sshProvider.runCommand("sudo apt install -y openjdk-11-jre-headless"); - if (exeCode != 0) - throw new IOException("Failed to install jdk on new VM"); - exeCode = sshProvider.runCommand("sudo apt install -y unzip"); - if (exeCode != 0) - throw new IOException("Failed to install unzip on new VM"); - exeCode = sshProvider.runCommand("wget https://github.com/apache/airavata-mft/releases/download/v0.0.1/MFT-Agent-0.01-bin.zip";); - if (exeCode != 0) - throw new IOException("Failed to download mft distribution"); - exeCode = sshProvider.runCommand("unzip MFT-Agent-0.01-bin.zip"); - if (exeCode != 0) - throw new IOException("Failed to unzip mft distribution"); + if (! preSavedImage) { + int exeCode = sshProvider.runCommand("sudo apt update -y"); + if (exeCode != 0) + throw new IOException("Failed to update apt for VM"); + exeCode = sshProvider.runCommand("sudo apt install -y openjdk-11-jre-headless"); + if (exeCode != 0) + throw new IOException("Failed to install jdk on new VM"); + exeCode = sshProvider.runCommand("sudo apt install -y unzip"); + if (exeCode != 0) + throw new IOException("Failed to install unzip on new VM"); + exeCode = sshProvider.runCommand("wget https://github.com/apache/airavata-mft/releases/download/v0.0.1/MFT-Agent-0.01-bin.zip";); + if (exeCode != 0) + throw new IOException("Failed to download mft distribution"); + exeCode = sshProvider.runCommand("unzip MFT-Agent-0.01-bin.zip"); + if (exeCode != 0) + throw new IOException("Failed to unzip mft distribution"); + } - exeCode = sshProvider.runCommand("sed -ir \"s/^[#]*\\s*agent.id=.*/agent.id=" + agentId + "/\" /home/ubuntu/MFT-Agent-0.01/conf/application.properties"); + int exeCode = sshProvider.runCommand("sed -ir \"s/^[#]*\\s*agent.id=.*/agent.id=" + agentId + "/\" /home/ubuntu/MFT-Agent-0.01/conf/application.properties"); if (exeCode != 0) throw new IOException("Failed to update agent id in config file"); @@ -279,7 +297,7 @@ public class EC2AgentSpawner extends AgentSpawner { } @Override - public void terminate() { + public void terminate(boolean failed) { terminateFuture = executor.submit(() -> { @@ -295,6 +313,13 @@ public class EC2AgentSpawner extends AgentSpawner { .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) .build(); + /*logger.info("Creating AMI for MFT Agent"); + CreateImageRequest createImageRequest = new CreateImageRequest().withName("mft-agent") + .withDescription("AMI For MFT Agent").withInstanceId(instanceId).withNoReboot(true) + .withBlockDeviceMappings(new BlockDeviceMapping().withEbs(new EbsBlockDevice())); + CreateImageResult imageCreateResult = amazonEC2.createImage(createImageRequest); + logger.info("Created AMI {} for instance {}", imageCreateResult.getImageId(), instanceId); + */ if (portForwardLock != null) { portForwardLock.countDown(); }
