This is an automated email from the ASF dual-hosted git repository. machristie pushed a commit to branch airavata-v2-refactoring in repository https://gitbox.apache.org/repos/asf/airavata.git
commit b182351a1619987637aa368d6d0e6858942ad2e8 Author: Marcus Christie <[email protected]> AuthorDate: Wed Jun 14 11:17:11 2023 -0400 Added updateExperiment method and tests to verify update logic --- .../airavata/apis/db/entity/ExperimentEntity.java | 7 +- .../airavata/apis/handlers/ExecutionHandler.java | 27 ++++ .../airavata/apis/mapper/ExperimentMapper.java | 8 +- .../apache/airavata/apis/mapper/ObjectMapper.java | 2 + .../apis/handlers/ExecutionHandlerTest.java | 146 +++++++++++++-------- .../main/proto/execution/experiment_service.proto | 8 ++ 6 files changed, 140 insertions(+), 58 deletions(-) diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ExperimentEntity.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ExperimentEntity.java index a8b1e73fc8..416a6a13ea 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ExperimentEntity.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/db/entity/ExperimentEntity.java @@ -78,8 +78,11 @@ public class ExperimentEntity extends BaseEntity { public void setRunConfigs(List<RunConfigurationEntity> runConfigs) { this.runConfigs = runConfigs; - for (RunConfigurationEntity runConfig : runConfigs) { - runConfig.setExperiment(this); + if (runConfigs != null) { + for (RunConfigurationEntity runConfig : runConfigs) { + runConfig.setExperiment(this); + } } + } } diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/handlers/ExecutionHandler.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/handlers/ExecutionHandler.java index 5736e8c83a..634dc41f2b 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/handlers/ExecutionHandler.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/handlers/ExecutionHandler.java @@ -5,11 +5,14 @@ import org.apache.airavata.api.execution.*; import org.apache.airavata.api.execution.stubs.Experiment; import org.apache.airavata.apis.db.entity.ExperimentEntity; import org.apache.airavata.apis.db.repository.ExperimentRepository; +import org.apache.airavata.apis.db.repository.RunConfigurationRepository; import org.apache.airavata.apis.mapper.ExperimentMapper; import org.apache.airavata.apis.scheduling.MetaScheduler; import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; +import java.util.Optional; + @GRpcService public class ExecutionHandler extends ExecutionServiceGrpc.ExecutionServiceImplBase { @@ -19,9 +22,13 @@ public class ExecutionHandler extends ExecutionServiceGrpc.ExecutionServiceImplB @Autowired ExperimentRepository experimentRepository; + @Autowired + RunConfigurationRepository runConfigurationRepository; + @Autowired ExperimentMapper experimentMapper; + // TODO: factor out database stuff into a transactional service layer @Override public void registerExperiment(ExperimentRegisterRequest request, StreamObserver<ExperimentRegisterResponse> responseObserver) { @@ -35,6 +42,26 @@ public class ExecutionHandler extends ExecutionServiceGrpc.ExecutionServiceImplB responseObserver.onCompleted(); } + @Override + public void updateExperiment(ExperimentUpdateRequest request, + StreamObserver<ExperimentUpdateResponse> responseObserver) { + + Experiment experiment = request.getExperiment(); + Optional<ExperimentEntity> maybeExperimentEntity = experimentRepository.findById(experiment.getExperimentId()); + // TODO: handle experiment not found + maybeExperimentEntity.ifPresent(entity -> { + // First delete any existing run configs + if (entity.getRunConfigs() != null && !entity.getRunConfigs().isEmpty()) { + runConfigurationRepository.deleteAll(entity.getRunConfigs()); + entity.setRunConfigs(null); + } + experimentMapper.mapModelToEntity(experiment, entity); + experimentRepository.save(entity); + }); + responseObserver.onNext(ExperimentUpdateResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } + @Override public void launchExperiment(ExperimentLaunchRequest request, StreamObserver<ExperimentLaunchResponse> responseObserver) { metaScheduler.scheduleExperiment(request); diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ExperimentMapper.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ExperimentMapper.java index eb96cdabfe..bb0c0ed24f 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ExperimentMapper.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ExperimentMapper.java @@ -14,15 +14,17 @@ public class ExperimentMapper implements ObjectMapper<ExperimentEntity, Experime @Override public Experiment mapEntityToModel(ExperimentEntity entity) { - return dozerMapper.map(entity, Experiment.class); } @Override public ExperimentEntity mapModelToEntity(Experiment model) { - ExperimentEntity entity = new ExperimentEntity(); + return mapModelToEntity(model, new ExperimentEntity()); + } + + @Override + public ExperimentEntity mapModelToEntity(Experiment model, ExperimentEntity entity) { dozerMapper.map(model, entity); return entity; } - } diff --git a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ObjectMapper.java b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ObjectMapper.java index 0f68d325df..9725f892c5 100644 --- a/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ObjectMapper.java +++ b/modules/airavata-apis/airavata-apis-server/src/main/java/org/apache/airavata/apis/mapper/ObjectMapper.java @@ -5,4 +5,6 @@ public interface ObjectMapper<E, M> { M mapEntityToModel(E entity); E mapModelToEntity(M model); + + E mapModelToEntity(M model, E entity); } diff --git a/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java b/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java index 26e66cd1ba..9a5dbe100e 100644 --- a/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java +++ b/modules/airavata-apis/airavata-apis-server/src/test/java/org/apache/airavata/apis/handlers/ExecutionHandlerTest.java @@ -2,7 +2,10 @@ package org.apache.airavata.apis.handlers; import org.apache.airavata.api.execution.ExperimentRegisterRequest; import org.apache.airavata.api.execution.ExperimentRegisterResponse; +import org.apache.airavata.api.execution.ExperimentUpdateRequest; +import org.apache.airavata.api.execution.ExperimentUpdateResponse; import org.apache.airavata.api.execution.stubs.*; +import org.apache.airavata.api.execution.stubs.Experiment.Builder; import org.apache.airavata.apis.db.entity.ApplicationRunInfoEntity; import org.apache.airavata.apis.db.entity.DataMovementConfigurationEntity; import org.apache.airavata.apis.db.entity.ExperimentEntity; @@ -22,6 +25,7 @@ import org.apache.airavata.apis.db.entity.backend.iface.SCPInterfaceEntity; import org.apache.airavata.apis.db.entity.backend.iface.SSHInterfaceEntity; import org.apache.airavata.apis.db.entity.data.InDataMovementEntity; import org.apache.airavata.apis.db.repository.ExperimentRepository; +import org.apache.airavata.apis.db.repository.RunConfigurationRepository; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -45,60 +49,61 @@ public class ExecutionHandlerTest { @Autowired ExperimentRepository experimentRepository; + @Autowired + RunConfigurationRepository runConfigurationRepository; + @Autowired EntityManager entityManager; + SSHInterface sshInterface = SSHInterface.newBuilder().setHostName("ssh-hostname").setPort(2222) + .setSshCredentialId("ssh-credential-id").build(); + SCPInterface scpInterface = SCPInterface.newBuilder().setHostName("scp-hostname").setPort(4422) + .setSshCredentialId("ssh-credential-id").build(); + ServerBackend serverBackend = ServerBackend.newBuilder().setHostName("server-hostname").setPort(1888) + .setCommandInterface(sshInterface).setDataInterface(scpInterface) + .setWorkingDirectory("/tmp/working-directory/some_wd/").build(); + // ApplicationInputs + CommandLineInput commandLineInput = CommandLineInput.newBuilder().setPosition(1).setPrefix("--prefix") + .setValue("input-value").build(); + ApplicationInput applicationInput = ApplicationInput.newBuilder().setIndex(1).setCommandLineInput(commandLineInput) + .setRequired(true).build(); + FileInput fileInput = FileInput.newBuilder().setFriendlyName("config file") + .setDestinationPath("/scratch/workdirs/W123/config.conf").build(); + ApplicationInput applicationInput2 = ApplicationInput.newBuilder().setIndex(2).setFileInput(fileInput) + .setRequired(true).build(); + EnvironmentInput environmentInput = EnvironmentInput.newBuilder().setKey("env-key").setValue("env-value").build(); + ApplicationInput applicationInput3 = ApplicationInput.newBuilder().setIndex(3).setEnvironmentInput(environmentInput) + .setRequired(false).build(); + // ApplicationOutputs + FileOutput fileOutput = FileOutput.newBuilder().setFriendlyName("output-file") + .setDestinationPath("/scratch/workdir/output.file").build(); + ApplicationOutput applicationOutput = ApplicationOutput.newBuilder().setIndex(1).setFileOutput(fileOutput) + .setRequired(true).build(); + StandardOut standardOut = StandardOut.newBuilder().setDestinationPath("/scratch/workdir/stdout").build(); + ApplicationOutput applicationOutput2 = ApplicationOutput.newBuilder().setIndex(2).setStdOut(standardOut) + .setRequired(false).build(); + StandardError standardError = StandardError.newBuilder().setDestinationPath("/scratch/workdir/stderr").build(); + ApplicationOutput applicationOutput3 = ApplicationOutput.newBuilder().setIndex(3).setStdErr(standardError) + .setRequired(false).build(); + Application application = Application.newBuilder().setName("test-application").addInputs(applicationInput) + .addInputs(applicationInput2).addInputs(applicationInput3).addOutputs(applicationOutput) + .addOutputs(applicationOutput2).addOutputs(applicationOutput3).build(); + ApplicationRunInfo applicationRunInfo = ApplicationRunInfo.newBuilder().setApplication(application).build(); + FileLocation sourceLocation = FileLocation.newBuilder().setStorageId("source-location-storage-id").build(); + InDataMovement inDataMovement = InDataMovement.newBuilder().setInputIndex(1).setSourceLocation(sourceLocation) + .build(); + DataMovementConfiguration dataMovementConfiguration = DataMovementConfiguration.newBuilder() + .addInMovements(inDataMovement).build(); + // TODO: add RunConfiguration for ec2 + // TODO: add RunConfiguration for local + RunConfiguration runConfiguration = RunConfiguration.newBuilder().setServer(serverBackend) + .setAppRunInfo(applicationRunInfo).addDataMovementConfigs(dataMovementConfiguration).build(); + Experiment experiment = Experiment.newBuilder().setCreationTime(System.currentTimeMillis()) + .setDescription("Sample Exp").setExperimentName("Exp Name").setGatewayId("gateway-id") + .setProjectId("project-id").addRunConfigs(runConfiguration).build(); + @Test - void testExperimentMapping() { - - SSHInterface sshInterface = SSHInterface.newBuilder().setHostName("ssh-hostname").setPort(2222) - .setSshCredentialId("ssh-credential-id").build(); - SCPInterface scpInterface = SCPInterface.newBuilder().setHostName("scp-hostname").setPort(4422) - .setSshCredentialId("ssh-credential-id").build(); - ServerBackend serverBackend = ServerBackend.newBuilder().setHostName("server-hostname").setPort(1888) - .setCommandInterface(sshInterface).setDataInterface(scpInterface) - .setWorkingDirectory("/tmp/working-directory/some_wd/").build(); - // ApplicationInputs - CommandLineInput commandLineInput = CommandLineInput.newBuilder().setPosition(1).setPrefix("--prefix") - .setValue("input-value").build(); - ApplicationInput applicationInput = ApplicationInput.newBuilder().setIndex(1) - .setCommandLineInput(commandLineInput).setRequired(true).build(); - FileInput fileInput = FileInput.newBuilder().setFriendlyName("config file") - .setDestinationPath("/scratch/workdirs/W123/config.conf").build(); - ApplicationInput applicationInput2 = ApplicationInput.newBuilder().setIndex(2).setFileInput(fileInput) - .setRequired(true).build(); - EnvironmentInput environmentInput = EnvironmentInput.newBuilder().setKey("env-key").setValue("env-value") - .build(); - ApplicationInput applicationInput3 = ApplicationInput.newBuilder().setIndex(3) - .setEnvironmentInput(environmentInput).setRequired(false).build(); - // ApplicationOutputs - FileOutput fileOutput = FileOutput.newBuilder().setFriendlyName("output-file") - .setDestinationPath("/scratch/workdir/output.file").build(); - ApplicationOutput applicationOutput = ApplicationOutput.newBuilder().setIndex(1).setFileOutput(fileOutput) - .setRequired(true).build(); - StandardOut standardOut = StandardOut.newBuilder().setDestinationPath("/scratch/workdir/stdout").build(); - ApplicationOutput applicationOutput2 = ApplicationOutput.newBuilder().setIndex(2).setStdOut(standardOut) - .setRequired(false).build(); - StandardError standardError = StandardError.newBuilder().setDestinationPath("/scratch/workdir/stderr").build(); - ApplicationOutput applicationOutput3 = ApplicationOutput.newBuilder().setIndex(3).setStdErr(standardError) - .setRequired(false).build(); - Application application = Application.newBuilder().setName("test-application").addInputs(applicationInput) - .addInputs(applicationInput2).addInputs(applicationInput3).addOutputs(applicationOutput) - .addOutputs(applicationOutput2).addOutputs(applicationOutput3) - .build(); - ApplicationRunInfo applicationRunInfo = ApplicationRunInfo.newBuilder().setApplication(application).build(); - FileLocation sourceLocation = FileLocation.newBuilder().setStorageId("source-location-storage-id").build(); - InDataMovement inDataMovement = InDataMovement.newBuilder().setInputIndex(1).setSourceLocation(sourceLocation) - .build(); - DataMovementConfiguration dataMovementConfiguration = DataMovementConfiguration.newBuilder() - .addInMovements(inDataMovement).build(); - // TODO: add RunConfiguration for ec2 - // TODO: add RunConfiguration for local - RunConfiguration runConfiguration = RunConfiguration.newBuilder().setServer(serverBackend) - .setAppRunInfo(applicationRunInfo).addDataMovementConfigs(dataMovementConfiguration).build(); - Experiment experiment = Experiment.newBuilder().setCreationTime(System.currentTimeMillis()) - .setDescription("Sample Exp").setExperimentName("Exp Name").setGatewayId("gateway-id") - .setProjectId("project-id").addRunConfigs(runConfiguration).build(); + void testRegisterExperiment() { ExperimentRegisterRequest experimentRegisterRequest = ExperimentRegisterRequest.newBuilder() .setExperiment(experiment).build(); @@ -108,9 +113,7 @@ public class ExecutionHandlerTest { assertTrue(responseObserver.isCompleted()); String experimentId = responseObserver.getNext().getExperimentId(); - // Force flushing experiment to database, then reload from database - entityManager.flush(); - entityManager.clear(); + flushAndClear(); ExperimentEntity experimentEntity = experimentRepository.findById(experimentId).get(); assertEquals(experiment.getCreationTime(), experimentEntity.getCreationTime()); @@ -212,4 +215,41 @@ public class ExecutionHandlerTest { assertEquals(inDataMovement.getSourceLocation().getStorageId(), inDataMovementEntity.getSourceLocation().getStorageId()); } + + @Test + public void testUpdateExperiment() { + + ExperimentRegisterRequest experimentRegisterRequest = ExperimentRegisterRequest.newBuilder() + .setExperiment(experiment).build(); + + TestStreamObserver<ExperimentRegisterResponse> responseObserver = new TestStreamObserver<>(); + executionHandler.registerExperiment(experimentRegisterRequest, responseObserver); + + assertTrue(responseObserver.isCompleted()); + String experimentId = responseObserver.getNext().getExperimentId(); + flushAndClear(); + + // Set the experiment id + Builder builder = experiment.toBuilder().setExperimentId(experimentId); + Experiment updatedExperiment = builder.build(); + + ExperimentUpdateRequest experimentUpdateRequest = ExperimentUpdateRequest.newBuilder() + .setExperiment(updatedExperiment).build(); + + TestStreamObserver<ExperimentUpdateResponse> updateResponseObserver = new TestStreamObserver<>(); + executionHandler.updateExperiment(experimentUpdateRequest, updateResponseObserver); + + assertTrue(responseObserver.isCompleted()); + flushAndClear(); + + // Just making sure that we only have one run configuration now + assertEquals(1, runConfigurationRepository.count()); + } + + void flushAndClear() { + // Force flushing experiment to database and removing from session so we can + // then reload from database + entityManager.flush(); + entityManager.clear(); + } } diff --git a/modules/airavata-apis/airavata-apis-stub/src/main/proto/execution/experiment_service.proto b/modules/airavata-apis/airavata-apis-stub/src/main/proto/execution/experiment_service.proto index 6370e4f963..48bea70f40 100644 --- a/modules/airavata-apis/airavata-apis-stub/src/main/proto/execution/experiment_service.proto +++ b/modules/airavata-apis/airavata-apis-stub/src/main/proto/execution/experiment_service.proto @@ -41,7 +41,15 @@ message ExperimentLaunchResponse { bool status = 1; } +message ExperimentUpdateRequest { + org.apache.airavata.api.auth.stubs.AuthzToken authz_token = 1; + org.apache.airavata.api.execution.stubs.Experiment experiment = 2; +} +message ExperimentUpdateResponse { +} + service ExecutionService { rpc registerExperiment(ExperimentRegisterRequest) returns (ExperimentRegisterResponse); rpc launchExperiment(ExperimentLaunchRequest) returns (ExperimentLaunchResponse); + rpc updateExperiment(ExperimentUpdateRequest) returns (ExperimentUpdateResponse); }
