This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch airavata-v2-refactoring in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 7ddbf1bbbe194a3a30427118792e0c4ae418c7dd Author: Marcus Christie <[email protected]> AuthorDate: Wed Jun 14 18:06:39 2023 -0400 Finish database mappings and tests --- .../apis/db/entity/ApplicationRunInfoEntity.java | 29 +++-- .../apache/airavata/apis/db/entity/BaseEntity.java | 12 +- .../application/runners/DockerRunnerEntity.java | 59 ++++++++- .../entity/application/runners/RunnerEntity.java | 13 ++ .../application/runners/SlurmRunnerEntity.java | 119 ++++++++++++++++++- .../apis/handlers/ExecutionHandlerTest.java | 132 ++++++++++++++++++--- 6 files changed, 329 insertions(+), 35 deletions(-) diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ApplicationRunInfoEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ApplicationRunInfoEntity.java index 62ff3847ae..cd313dcff9 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ApplicationRunInfoEntity.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ApplicationRunInfoEntity.java @@ -2,6 +2,7 @@ package org.apache.airavata.apis.db.entity; import org.apache.airavata.apis.db.entity.application.ApplicationEntity; import org.apache.airavata.apis.db.entity.application.runners.DockerRunnerEntity; +import org.apache.airavata.apis.db.entity.application.runners.RunnerEntity; import org.apache.airavata.apis.db.entity.application.runners.SlurmRunnerEntity; import javax.persistence.CascadeType; @@ -18,13 +19,8 @@ public class ApplicationRunInfoEntity extends BaseEntity { private ApplicationEntity application; @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "docker_runner_id") - DockerRunnerEntity dockerRunner; - - @OneToOne(cascade = CascadeType.ALL) - @JoinColumn(name = "slurm_runner_id") - SlurmRunnerEntity slurmRunner; - + @JoinColumn(name = "runner_id") + RunnerEntity runner; public ApplicationEntity getApplication() { return application; @@ -34,20 +30,31 @@ public class ApplicationRunInfoEntity extends BaseEntity { this.application = application; } + public RunnerEntity getRunner() { + return runner; + } + + public void setRunner(RunnerEntity runner) { + this.runner = runner; + } + + /* + * Helper getters/setters for mapping from/to protobuf messages + */ public DockerRunnerEntity getDockerRunner() { - return dockerRunner; + return runner instanceof DockerRunnerEntity ? (DockerRunnerEntity) runner : null; } public void setDockerRunner(DockerRunnerEntity dockerRunner) { - this.dockerRunner = dockerRunner; + this.runner = dockerRunner; } public SlurmRunnerEntity getSlurmRunner() { - return slurmRunner; + return runner instanceof SlurmRunnerEntity ? (SlurmRunnerEntity) runner : null; } public void setSlurmRunner(SlurmRunnerEntity slurmRunner) { - this.slurmRunner = slurmRunner; + this.runner = slurmRunner; } } diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/BaseEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/BaseEntity.java index 18a99f8819..af274409e8 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/BaseEntity.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/BaseEntity.java @@ -25,10 +25,7 @@ import org.springframework.data.annotation.CreatedDate; import org.springframework.data.annotation.LastModifiedDate; import org.springframework.data.jpa.domain.support.AuditingEntityListener; -import javax.persistence.Column; -import javax.persistence.EntityListeners; -import javax.persistence.Id; -import javax.persistence.MappedSuperclass; +import javax.persistence.*; import java.time.Instant; import java.util.UUID; @@ -49,6 +46,13 @@ public abstract class BaseEntity { @LastModifiedDate Instant lastModifiedAt; + // Version column is needed to determine if entity has been saved or not. + // Normally if the 'id' is null, JPA will assume that it hasn't been saved, but + // 'id' is never null even for a non-persisted entity. With the Version column, + // a null Version value indicates that the entity hasn't been persisted. + @Version + Integer entityVersion; + public String getId() { return id; } diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/DockerRunnerEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/DockerRunnerEntity.java index 6346c03f32..eb2154eb8f 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/DockerRunnerEntity.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/DockerRunnerEntity.java @@ -1,10 +1,63 @@ package org.apache.airavata.apis.db.entity.application.runners; -import org.apache.airavata.apis.db.entity.BaseEntity; - +import javax.persistence.Column; import javax.persistence.Entity; @Entity -public class DockerRunnerEntity extends BaseEntity { +public class DockerRunnerEntity extends RunnerEntity { + + @Column + String imageName; + + @Column + String imageTag; + + @Column + String repository; + + @Column + String dockerCredentialId; + + @Column + String runCommand; + + public String getImageName() { + return imageName; + } + + public void setImageName(String imageName) { + this.imageName = imageName; + } + + public String getImageTag() { + return imageTag; + } + + public void setImageTag(String imageTag) { + this.imageTag = imageTag; + } + + public String getRepository() { + return repository; + } + + public void setRepository(String repository) { + this.repository = repository; + } + + public String getDockerCredentialId() { + return dockerCredentialId; + } + + public void setDockerCredentialId(String dockerCredentialId) { + this.dockerCredentialId = dockerCredentialId; + } + + public String getRunCommand() { + return runCommand; + } + public void setRunCommand(String runCommand) { + this.runCommand = runCommand; + } } diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/RunnerEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/RunnerEntity.java new file mode 100644 index 0000000000..4e8e822eb6 --- /dev/null +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/RunnerEntity.java @@ -0,0 +1,13 @@ +package org.apache.airavata.apis.db.entity.application.runners; + +import org.apache.airavata.apis.db.entity.BaseEntity; + +import javax.persistence.Entity; +import javax.persistence.Inheritance; +import javax.persistence.InheritanceType; + +@Entity +@Inheritance(strategy = InheritanceType.TABLE_PER_CLASS) +public abstract class RunnerEntity extends BaseEntity { + +} diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/SlurmRunnerEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/SlurmRunnerEntity.java index 05c0b70124..e0a18a8aea 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/SlurmRunnerEntity.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/application/runners/SlurmRunnerEntity.java @@ -1,9 +1,122 @@ package org.apache.airavata.apis.db.entity.application.runners; -import org.apache.airavata.apis.db.entity.BaseEntity; - +import javax.persistence.Column; +import javax.persistence.ElementCollection; import javax.persistence.Entity; +import java.util.List; + @Entity -public class SlurmRunnerEntity extends BaseEntity { +public class SlurmRunnerEntity extends RunnerEntity { + + @Column + int nodes; + + @Column + int cpus; + + @Column + int memory; + + @Column + int wallTime; + + @ElementCollection + List<String> preJobCommands; + + @ElementCollection + List<String> moduleLoadCommands; + + @Column + String executable; + + @ElementCollection + List<String> postJobCommands; + + @Column + String queue; + + @ElementCollection + List<String> notificationEmails; + + public int getNodes() { + return nodes; + } + + public void setNodes(int nodes) { + this.nodes = nodes; + } + + public int getCpus() { + return cpus; + } + + public void setCpus(int cpus) { + this.cpus = cpus; + } + + public int getMemory() { + return memory; + } + + public void setMemory(int memory) { + this.memory = memory; + } + + public int getWallTime() { + return wallTime; + } + + public void setWallTime(int wallTime) { + this.wallTime = wallTime; + } + + public List<String> getPreJobCommands() { + return preJobCommands; + } + + public void setPreJobCommands(List<String> preJobCommands) { + this.preJobCommands = preJobCommands; + } + + public List<String> getModuleLoadCommands() { + return moduleLoadCommands; + } + + public void setModuleLoadCommands(List<String> moduleLoadCommands) { + this.moduleLoadCommands = moduleLoadCommands; + } + + public String getExecutable() { + return executable; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public List<String> getPostJobCommands() { + return postJobCommands; + } + + public void setPostJobCommands(List<String> postJobCommands) { + this.postJobCommands = postJobCommands; + } + + public String getQueue() { + return queue; + } + + public void setQueue(String queue) { + this.queue = queue; + } + + public List<String> getNotificationEmails() { + return notificationEmails; + } + + public void setNotificationEmails(List<String> notificationEmails) { + this.notificationEmails = notificationEmails; + } + } diff --git a/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java b/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java index 9a5dbe100e..e8494c4721 100644 --- a/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java +++ b/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java @@ -1,5 +1,6 @@ package org.apache.airavata.apis.handlers; +import com.google.protobuf.InvalidProtocolBufferException; import org.apache.airavata.api.execution.ExperimentRegisterRequest; import org.apache.airavata.api.execution.ExperimentRegisterResponse; import org.apache.airavata.api.execution.ExperimentUpdateRequest; @@ -19,11 +20,16 @@ import org.apache.airavata.apis.db.entity.application.output.ApplicationOutputEn import org.apache.airavata.apis.db.entity.application.output.FileOutputEntity; import org.apache.airavata.apis.db.entity.application.output.StandardErrorEntity; import org.apache.airavata.apis.db.entity.application.output.StandardOutEntity; +import org.apache.airavata.apis.db.entity.application.runners.DockerRunnerEntity; +import org.apache.airavata.apis.db.entity.application.runners.SlurmRunnerEntity; import org.apache.airavata.apis.db.entity.backend.ComputeBackendEntity; +import org.apache.airavata.apis.db.entity.backend.EC2BackendEntity; +import org.apache.airavata.apis.db.entity.backend.LocalBackendEntity; import org.apache.airavata.apis.db.entity.backend.ServerBackendEntity; import org.apache.airavata.apis.db.entity.backend.iface.SCPInterfaceEntity; import org.apache.airavata.apis.db.entity.backend.iface.SSHInterfaceEntity; import org.apache.airavata.apis.db.entity.data.InDataMovementEntity; +import org.apache.airavata.apis.db.entity.data.OutDataMovementEntity; import org.apache.airavata.apis.db.repository.ExperimentRepository; import org.apache.airavata.apis.db.repository.RunConfigurationRepository; import org.junit.jupiter.api.Test; @@ -35,6 +41,7 @@ import javax.persistence.EntityManager; import java.util.Optional; +import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -88,19 +95,44 @@ public class ExecutionHandlerTest { Application application = Application.newBuilder().setName("test-application").addInputs(applicationInput) .addInputs(applicationInput2).addInputs(applicationInput3).addOutputs(applicationOutput) .addOutputs(applicationOutput2).addOutputs(applicationOutput3).build(); - ApplicationRunInfo applicationRunInfo = ApplicationRunInfo.newBuilder().setApplication(application).build(); - FileLocation sourceLocation = FileLocation.newBuilder().setStorageId("source-location-storage-id").build(); + DockerRunner dockerRunner = DockerRunner.newBuilder().setImageName("docker-image-name") + .setImageTag("docker-image-tag").setRepository("docker-repository") + .setDockerCredentialId("docker-credential-id").setRunCommand("docker-run-command").build(); + SlurmRunner slurmRunner = SlurmRunner.newBuilder().setNodes(3).setCpus(7).setMemory(23).setWallTime(37) + .addAllPreJobCommands(asList("prejob1", "prejob2", "prejob3")) + .addAllModuleLoadCommands(asList("module1", "module2", "module3")).setExecutable("/path/to/executable") + .addAllPostJobCommands(asList("postjob1", "postjob2", "postjob13")).setQueue("shared") + .addAllNotificationEmails(asList("[email protected]", "[email protected]")).build(); + ApplicationRunInfo applicationRunInfo = ApplicationRunInfo.newBuilder().setApplication(application) + .setDockerRunner(dockerRunner).build(); + ApplicationRunInfo applicationRunInfoSlurmRunner = ApplicationRunInfo.newBuilder() + .setApplication(cloneApplication(application)).setSlurmRunner(slurmRunner).build(); + FileLocation sourceLocation = FileLocation.newBuilder().setStorageId("source-location-storage-id") + .setPath("/path/to/source").setStorageCredentialId("source-storage-credential-id").build(); InDataMovement inDataMovement = InDataMovement.newBuilder().setInputIndex(1).setSourceLocation(sourceLocation) .build(); + + FileLocation destinationLocation = FileLocation.newBuilder().setStorageId("destination-location-storage-id") + .setPath("/path/to/destination").setStorageCredentialId("dest-storage-credential-id").build(); + OutDataMovement outDataMovement = OutDataMovement.newBuilder().setOutputIndex(2) + .setDestinationLocation(destinationLocation).build(); DataMovementConfiguration dataMovementConfiguration = DataMovementConfiguration.newBuilder() - .addInMovements(inDataMovement).build(); - // TODO: add RunConfiguration for ec2 - // TODO: add RunConfiguration for local - RunConfiguration runConfiguration = RunConfiguration.newBuilder().setServer(serverBackend) + .addInMovements(inDataMovement).addOutMovements(outDataMovement).build(); + EC2Backend ec2Backend = EC2Backend.newBuilder().setFlavor("t1.micro").setRegion("us-east-1") + .setAwsCredentialId("aws-credential-id").build(); + LocalBackend localBackend = LocalBackend.newBuilder().setAgentId("agent-id").setAgentTokenId("agent-token-id") + .build(); + RunConfiguration runConfigurationServer = RunConfiguration.newBuilder().setServer(serverBackend) .setAppRunInfo(applicationRunInfo).addDataMovementConfigs(dataMovementConfiguration).build(); + RunConfiguration runConfigurationEC2Backend = cloneRunConfiguration(runConfigurationServer).toBuilder() + .setEc2(ec2Backend).setAppRunInfo(applicationRunInfoSlurmRunner).build(); + RunConfiguration runConfigurationLocalBackend = cloneRunConfiguration(runConfigurationServer).toBuilder() + .setLocal(localBackend).build(); + Experiment experiment = Experiment.newBuilder().setCreationTime(System.currentTimeMillis()) .setDescription("Sample Exp").setExperimentName("Exp Name").setGatewayId("gateway-id") - .setProjectId("project-id").addRunConfigs(runConfiguration).build(); + .setProjectId("project-id").addRunConfigs(runConfigurationServer).addRunConfigs(runConfigurationEC2Backend) + .addRunConfigs(runConfigurationLocalBackend).build(); @Test void testRegisterExperiment() { @@ -124,10 +156,15 @@ public class ExecutionHandlerTest { // RunConfiguration assertEquals(experiment.getRunConfigsCount(), experimentEntity.getRunConfigs().size()); - RunConfigurationEntity runConfigEntity = experimentEntity.getRunConfigs().get(0); + RunConfigurationEntity runConfigWithServerEntity = experimentEntity.getRunConfigs().stream() + .filter(rc -> rc.getServer() != null).findFirst().get(); + RunConfigurationEntity runConfigWithEC2Entity = experimentEntity.getRunConfigs().stream() + .filter(rc -> rc.getEc2() != null).findFirst().get(); + RunConfigurationEntity runConfigWithLocalBackendEntity = experimentEntity.getRunConfigs().stream() + .filter(rc -> rc.getLocal() != null).findFirst().get(); // ComputeBackend - ComputeBackendEntity computeBackendEntity = runConfigEntity.getComputeBackend(); + ComputeBackendEntity computeBackendEntity = runConfigWithServerEntity.getComputeBackend(); assertTrue(computeBackendEntity instanceof ServerBackendEntity); ServerBackendEntity serverBackendEntity = (ServerBackendEntity) computeBackendEntity; assertEquals(serverBackend.getHostName(), serverBackendEntity.getHostName()); @@ -142,8 +179,21 @@ public class ExecutionHandlerTest { assertEquals(scpInterface.getPort(), scpInterfaceEntity.getPort().intValue()); assertEquals(scpInterface.getSshCredentialId(), scpInterfaceEntity.getSshCredentialId()); + // EC2Backend + EC2BackendEntity ec2BackendEntity = runConfigWithEC2Entity.getEc2(); + assertTrue(runConfigWithEC2Entity.getComputeBackend() instanceof EC2BackendEntity); + assertEquals(ec2Backend.getFlavor(), ec2BackendEntity.getFlavor()); + assertEquals(ec2Backend.getRegion(), ec2BackendEntity.getRegion()); + assertEquals(ec2Backend.getAwsCredentialId(), ec2BackendEntity.getAwsCredentialId()); + + // LocalBackend + LocalBackendEntity localBackendEntity = runConfigWithLocalBackendEntity.getLocal(); + assertTrue(runConfigWithLocalBackendEntity.getComputeBackend() instanceof LocalBackendEntity); + assertEquals(localBackend.getAgentId(), localBackendEntity.getAgentId()); + assertEquals(localBackend.getAgentTokenId(), localBackendEntity.getAgentTokenId()); + // ApplicationRunInfo - ApplicationRunInfoEntity applicationRunInfoEntity = runConfigEntity.getAppRunInfo(); + ApplicationRunInfoEntity applicationRunInfoEntity = runConfigWithServerEntity.getAppRunInfo(); ApplicationEntity applicationEntity = applicationRunInfoEntity.getApplication(); assertEquals(application.getName(), applicationEntity.getName()); assertEquals(application.getInputsCount(), applicationEntity.getInputs().size()); @@ -202,10 +252,32 @@ public class ExecutionHandlerTest { assertNotNull(standardErrorEntity); assertEquals(standardError.getDestinationPath(), standardErrorEntity.getDestinationPath()); + // DockerRunner + DockerRunnerEntity dockerRunnerEntity = applicationRunInfoEntity.getDockerRunner(); + assertEquals(dockerRunner.getImageName(), dockerRunnerEntity.getImageName()); + assertEquals(dockerRunner.getImageTag(), dockerRunnerEntity.getImageTag()); + assertEquals(dockerRunner.getRepository(), dockerRunnerEntity.getRepository()); + assertEquals(dockerRunner.getDockerCredentialId(), dockerRunnerEntity.getDockerCredentialId()); + assertEquals(dockerRunner.getRunCommand(), dockerRunnerEntity.getRunCommand()); + + // SlurmRunner + SlurmRunnerEntity slurmRunnerEntity = runConfigWithEC2Entity.getAppRunInfo().getSlurmRunner(); + assertEquals(slurmRunner.getNodes(), slurmRunnerEntity.getNodes()); + assertEquals(slurmRunner.getCpus(), slurmRunnerEntity.getCpus()); + assertEquals(slurmRunner.getMemory(), slurmRunnerEntity.getMemory()); + assertEquals(slurmRunner.getWallTime(), slurmRunnerEntity.getWallTime()); + assertEquals(slurmRunner.getPreJobCommandsList(), slurmRunnerEntity.getPreJobCommands()); + assertEquals(slurmRunner.getModuleLoadCommandsList(), slurmRunnerEntity.getModuleLoadCommands()); + assertEquals(slurmRunner.getExecutable(), slurmRunnerEntity.getExecutable()); + assertEquals(slurmRunner.getPostJobCommandsList(), slurmRunnerEntity.getPostJobCommands()); + assertEquals(slurmRunner.getQueue(), slurmRunnerEntity.getQueue()); + assertEquals(slurmRunner.getNotificationEmailsList(), slurmRunnerEntity.getNotificationEmails()); + // DataMovementConfiguration - assertEquals(runConfiguration.getDataMovementConfigsCount(), runConfigEntity.getDataMovementConfigs().size()); - DataMovementConfigurationEntity dataMovementConfigurationEntity = runConfigEntity.getDataMovementConfigs() - .get(0); + assertEquals(runConfigurationServer.getDataMovementConfigsCount(), + runConfigWithServerEntity.getDataMovementConfigs().size()); + DataMovementConfigurationEntity dataMovementConfigurationEntity = runConfigWithServerEntity + .getDataMovementConfigs().get(0); assertEquals(dataMovementConfiguration.getInMovementsCount(), dataMovementConfigurationEntity.getInMovements().size()); @@ -214,6 +286,21 @@ public class ExecutionHandlerTest { assertEquals(inDataMovement.getInputIndex(), inDataMovementEntity.getInputIndex()); assertEquals(inDataMovement.getSourceLocation().getStorageId(), inDataMovementEntity.getSourceLocation().getStorageId()); + assertEquals(inDataMovement.getSourceLocation().getPath(), inDataMovementEntity.getSourceLocation().getPath()); + assertEquals(inDataMovement.getSourceLocation().getStorageCredentialId(), + inDataMovementEntity.getSourceLocation().getStorageCredentialId()); + + // OutDataMovement + OutDataMovementEntity outDataMovementEntity = dataMovementConfigurationEntity.getOutMovements().iterator() + .next(); + assertEquals(outDataMovement.getOutputIndex(), outDataMovementEntity.getOutputIndex()); + assertEquals(outDataMovement.getDestinationLocation().getStorageId(), + outDataMovementEntity.getDestinationLocation().getStorageId()); + assertEquals(outDataMovement.getDestinationLocation().getPath(), + outDataMovementEntity.getDestinationLocation().getPath()); + assertEquals(outDataMovement.getDestinationLocation().getStorageCredentialId(), + outDataMovementEntity.getDestinationLocation().getStorageCredentialId()); + } @Test @@ -243,7 +330,7 @@ public class ExecutionHandlerTest { flushAndClear(); // Just making sure that we only have one run configuration now - assertEquals(1, runConfigurationRepository.count()); + assertEquals(experiment.getRunConfigsCount(), runConfigurationRepository.count()); } void flushAndClear() { @@ -252,4 +339,21 @@ public class ExecutionHandlerTest { entityManager.flush(); entityManager.clear(); } + + RunConfiguration cloneRunConfiguration(RunConfiguration runConfiguration) { + + try { + return RunConfiguration.parseFrom(runConfiguration.toByteArray()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + + Application cloneApplication(Application application) { + try { + return Application.parseFrom(application.toByteArray()); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } }
