This is an automated email from the ASF dual-hosted git repository. dimuthuupe pushed a commit to branch file-server in repository https://gitbox.apache.org/repos/asf/airavata.git
commit e18975880ce038d205a9046b406fea11e8b58079 Author: DImuthuUpe <[email protected]> AuthorDate: Mon Dec 16 08:11:04 2024 -0500 File server initial implementation --- modules/agent-framework/connection-service/pom.xml | 3 +- .../airavata/helix/impl/task/AiravataTask.java | 2 +- modules/file-server/pom.xml | 81 +++++++++++++++ .../main/assembly/file-service-bin-assembly.xml | 90 +++++++++++++++++ .../file/server/FileServerApplication.java | 16 +++ .../file/server/FileServerConfiguration.java | 47 +++++++++ .../file/server/controller/FileController.java | 87 ++++++++++++++++ .../file/server/model/AiravataDirectory.java | 49 +++++++++ .../airavata/file/server/model/AiravataFile.java | 47 +++++++++ .../file/server/model/FileUploadResponse.java | 47 +++++++++ .../file/server/service/AirvataFileService.java | 88 ++++++++++++++++ .../file/server/service/ProcessDataManager.java | 57 +++++++++++ .../src/main/resources/application.properties | 12 +++ .../distribution/bin/file-service-daemon.sh | 112 +++++++++++++++++++++ .../resources/distribution/bin/file-service.sh | 70 +++++++++++++ .../src/main/resources/distribution/bin/setenv.sh | 46 +++++++++ .../distribution/conf/application.properties | 12 +++ .../main/resources/distribution/conf/log4j2.xml | 53 ++++++++++ pom.xml | 2 + 19 files changed, 918 insertions(+), 3 deletions(-) diff --git a/modules/agent-framework/connection-service/pom.xml b/modules/agent-framework/connection-service/pom.xml index 619c872692..3ee06a2ccb 100644 --- a/modules/agent-framework/connection-service/pom.xml +++ b/modules/agent-framework/connection-service/pom.xml @@ -15,7 +15,6 @@ <maven.compiler.source>11</maven.compiler.source> <maven.compiler.target>11</maven.compiler.target> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <spring.boot.version>3.2.4</spring.boot.version> <jul-to-slf4j.version>1.7.0</jul-to-slf4j.version> <json.version>20240303</json.version> <javax.version>2.0.1.Final</javax.version> @@ -159,4 +158,4 @@ </plugins> </build> -</project> \ No newline at end of file +</project> diff --git a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java index d2208d5293..0715a17bdb 100644 --- a/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java +++ b/modules/airavata-helix/helix-spectator/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java @@ -496,7 +496,7 @@ public abstract class AiravataTask extends AbstractTask { } } - private void loadContext() throws TaskOnFailException { + protected void loadContext() throws TaskOnFailException { try { logger.info("Loading context for task " + getTaskId()); processModel = getRegistryServiceClient().getProcess(processId); diff --git a/modules/file-server/pom.xml b/modules/file-server/pom.xml new file mode 100644 index 0000000000..a7065bdff3 --- /dev/null +++ b/modules/file-server/pom.xml @@ -0,0 +1,81 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.airavata</groupId> + <artifactId>airavata</artifactId> + <version>0.21-SNAPSHOT</version> + <relativePath>../../pom.xml</relativePath> + </parent> + <artifactId>file-server</artifactId> + <name>airavata-file-server</name> + + <dependencies> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-web</artifactId> + <version>${spring.boot.version}</version> + <exclusions> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-to-slf4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>task-core</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.airavata</groupId> + <artifactId>helix-spectator</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.8.1</version> + <configuration> + <compilerArgs> + <arg>-parameters</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <version>${maven.assembly.plugin}</version> + <executions> + <execution> + <id>file-service-distribution-package</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <tarLongFileMode>posix</tarLongFileMode> + <finalName>${file.service.dist.name}</finalName> + <descriptors> + <descriptor>src/main/assembly/file-service-bin-assembly.xml</descriptor> + </descriptors> + <attach>false</attach> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <properties> + <file.service.dist.name>File-Service-0.01</file.service.dist.name> + </properties> +</project> diff --git a/modules/file-server/src/main/assembly/file-service-bin-assembly.xml b/modules/file-server/src/main/assembly/file-service-bin-assembly.xml new file mode 100644 index 0000000000..8d72d47391 --- /dev/null +++ b/modules/file-server/src/main/assembly/file-service-bin-assembly.xml @@ -0,0 +1,90 @@ + +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<!DOCTYPE assembly [ + <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*> + <!ELEMENT id (#PCDATA)> + <!ELEMENT includeBaseDirectory (#PCDATA)> + <!ELEMENT baseDirectory (#PCDATA)> + <!ELEMENT formats (format)*> + <!ELEMENT format (#PCDATA)> + <!ELEMENT fileSets (fileSet)*> + <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*> + <!ELEMENT directory (#PCDATA)> + <!ELEMENT outputDirectory (#PCDATA)> + <!ELEMENT includes (include)*> + <!ELEMENT include (#PCDATA)> + <!ELEMENT dependencySets (dependencySet)*> + <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*> + ]> +<assembly> + <id>bin</id> + <includeBaseDirectory>true</includeBaseDirectory> + <baseDirectory>${file.service.dist.name}</baseDirectory> + <formats> + <format>tar.gz</format> + <format>zip</format> + </formats> + + <fileSets> + <fileSet> + <directory>src/main/resources/distribution/bin</directory> + <outputDirectory>bin</outputDirectory> + <fileMode>777</fileMode> + <includes> + <include>*.sh</include> + </includes> + </fileSet> + <fileSet> + <directory>src/main/resources/distribution/conf</directory> + <outputDirectory>conf</outputDirectory> + <includes> + <include>application.properties</include> + <include>log4j2.xml</include> + </includes> + </fileSet> + <fileSet> + <directory>./</directory> + <outputDirectory>logs</outputDirectory> + <excludes> + <exclude>*/**</exclude> + </excludes> + </fileSet> + <fileSet> + <directory>target</directory> + <outputDirectory>lib</outputDirectory> + <includes> + <include>*.jar</include> + </includes> + </fileSet> + </fileSets> + + <dependencySets> + <dependencySet> + <useProjectArtifact>false</useProjectArtifact> + <outputDirectory>lib</outputDirectory> + <includes> + <include>*</include> + </includes> + </dependencySet> + </dependencySets> + +</assembly> diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java new file mode 100644 index 0000000000..42bf099184 --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerApplication.java @@ -0,0 +1,16 @@ +package org.apache.airavata.file.server; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.groovy.template.GroovyTemplateAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.ComponentScan; + +@SpringBootApplication(scanBasePackages = "org.apache.airavata.file.server", + exclude = {GroovyTemplateAutoConfiguration.class}) +@EnableConfigurationProperties(FileServerConfiguration.class) +public class FileServerApplication { + public static void main(String[] args) { + SpringApplication.run(FileServerApplication.class, args); + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java new file mode 100644 index 0000000000..e4b6df32d8 --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/FileServerConfiguration.java @@ -0,0 +1,47 @@ +package org.apache.airavata.file.server; + +import org.apache.airavata.common.utils.ServerSettings; +import org.apache.airavata.common.utils.ThriftClientPool; +import org.apache.airavata.helix.core.support.adaptor.AdaptorSupportImpl; +import org.apache.airavata.helix.task.api.support.AdaptorSupport; +import org.apache.airavata.registry.api.RegistryService; +import org.apache.commons.pool2.impl.GenericObjectPoolConfig; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties +public class FileServerConfiguration { + + @Bean + public AdaptorSupport adaptorSupport() { + return AdaptorSupportImpl.getInstance(); + } + + //regserver.server.host + @Value("${regserver.server.host:localhost}") + private String registryServerHost; + //regserver.server.port + + @Value("${regserver.server.port:8970}") + private int registryServerPort; + + @Bean + public ThriftClientPool<RegistryService.Client> registryClientPool() { + GenericObjectPoolConfig<RegistryService.Client> poolConfig = new GenericObjectPoolConfig<>(); + poolConfig.setMaxTotal(100); + poolConfig.setMinIdle(5); + poolConfig.setBlockWhenExhausted(true); + poolConfig.setTestOnBorrow(true); + poolConfig.setTestWhileIdle(true); + // must set timeBetweenEvictionRunsMillis since eviction doesn't run unless that is positive + poolConfig.setTimeBetweenEvictionRunsMillis(5L * 60L * 1000L); + poolConfig.setNumTestsPerEvictionRun(10); + poolConfig.setMaxWaitMillis(3000); + + return new ThriftClientPool<>( + RegistryService.Client::new, poolConfig, registryServerHost, registryServerPort); + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java new file mode 100644 index 0000000000..b735da7765 --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/controller/FileController.java @@ -0,0 +1,87 @@ +package org.apache.airavata.file.server.controller; + +import org.apache.airavata.file.server.model.AiravataDirectory; +import org.apache.airavata.file.server.model.FileUploadResponse; +import org.apache.airavata.file.server.service.AirvataFileService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.Resource; +import org.springframework.core.io.UrlResource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.servlet.support.ServletUriComponentsBuilder; + +import java.io.File; +import java.nio.file.Path; + +@Controller +public class FileController { + + private final static Logger logger = LoggerFactory.getLogger(FileController.class); + + @Autowired + private AirvataFileService fileService; + + @GetMapping("/list/{live}/{processId}") + @ResponseBody + public Object listFilesRoot(@PathVariable String live, + @PathVariable String processId) throws Exception { + try { + return fileService.listFiles(processId, ""); + } catch (Exception e) { + logger.error("Failed to list files for path {} in process {}", "root path", processId, e); + throw e; + } + } + @GetMapping("/list/{live}/{processId}/{subPath}") + @ResponseBody + public Object listFiles(@PathVariable String live, + @PathVariable String processId, + @PathVariable String subPath) throws Exception { + try { + return fileService.listFiles(processId, subPath); + } catch (Exception e) { + logger.error("Failed to list files for path {} in process {}", subPath, processId, e); + throw e; + } + } + + @GetMapping("/download/{isLive}/{processId}/{subPath}") + @ResponseBody + public ResponseEntity downloadFile(@PathVariable String isLive, + @PathVariable String processId, + @PathVariable String subPath) { + + try { + Path localPath = fileService.downloadFile(processId, subPath); + Resource resource = new UrlResource(localPath.toUri()); + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, + "attachment; filename=\"" + new File(subPath).getName() + "\"") + .body(resource); + } catch (Exception e) { + return ResponseEntity.internalServerError() + .body("An internal server error occurred: " + e.getMessage()); + } + } + + + @PostMapping("/upload-file") + @ResponseBody + public FileUploadResponse uploadFile(@RequestParam("file") MultipartFile file) { + String name = ""; + + String uri = ServletUriComponentsBuilder.fromCurrentContextPath() + .path("/download/") + .path(name) + .toUriString(); + + return new FileUploadResponse(name, uri, file.getContentType(), file.getSize()); + } + + +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java new file mode 100644 index 0000000000..bed98f1ffb --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataDirectory.java @@ -0,0 +1,49 @@ +package org.apache.airavata.file.server.model; + +import java.util.ArrayList; +import java.util.List; + +public class AiravataDirectory { + private String directoryName; + private long createdTime; + + private List<AiravataFile> innerFiles = new ArrayList<>(); + private List<AiravataDirectory> innerDirectories = new ArrayList<>(); + + public AiravataDirectory(String directoryName, long createdTime) { + this.directoryName = directoryName; + this.createdTime = createdTime; + } + + public String getDirectoryName() { + return directoryName; + } + + public void setDirectoryName(String directoryName) { + this.directoryName = directoryName; + } + + public long getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(long createdTime) { + this.createdTime = createdTime; + } + + public List<AiravataFile> getInnerFiles() { + return innerFiles; + } + + public void setInnerFiles(List<AiravataFile> innerFiles) { + this.innerFiles = innerFiles; + } + + public List<AiravataDirectory> getInnerDirectories() { + return innerDirectories; + } + + public void setInnerDirectories(List<AiravataDirectory> innerDirectories) { + this.innerDirectories = innerDirectories; + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java new file mode 100644 index 0000000000..f2a174f6a7 --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/AiravataFile.java @@ -0,0 +1,47 @@ +package org.apache.airavata.file.server.model; + +public class AiravataFile { + private String fileName; + private long fileSize; + private long createdTime; + private long updatedTime; + + public AiravataFile(String fileName, long fileSize, long createdTime, long updatedTime) { + this.fileName = fileName; + this.fileSize = fileSize; + this.createdTime = createdTime; + this.updatedTime = updatedTime; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public long getFileSize() { + return fileSize; + } + + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + + public long getCreatedTime() { + return createdTime; + } + + public void setCreatedTime(long createdTime) { + this.createdTime = createdTime; + } + + public long getUpdatedTime() { + return updatedTime; + } + + public void setUpdatedTime(long updatedTime) { + this.updatedTime = updatedTime; + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java new file mode 100644 index 0000000000..0f061cad2b --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/model/FileUploadResponse.java @@ -0,0 +1,47 @@ +package org.apache.airavata.file.server.model; + +public class FileUploadResponse { + private String name; + private String uri; + private String type; + private long size; + + public FileUploadResponse(String name, String uri, String type, long size) { + this.name = name; + this.uri = uri; + this.type = type; + this.size = size; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUri() { + return uri; + } + + public void setUri(String uri) { + this.uri = uri; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java new file mode 100644 index 0000000000..949d59c77b --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/AirvataFileService.java @@ -0,0 +1,88 @@ +package org.apache.airavata.file.server.service; + +import org.apache.airavata.agents.api.AgentAdaptor; +import org.apache.airavata.agents.api.FileMetadata; +import org.apache.airavata.common.utils.ThriftClientPool; +import org.apache.airavata.file.server.model.AiravataDirectory; +import org.apache.airavata.file.server.model.AiravataFile; +import org.apache.airavata.helix.task.api.support.AdaptorSupport; +import org.apache.airavata.registry.api.RegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +@Service +public class AirvataFileService { + + private final static Logger logger = LoggerFactory.getLogger(AirvataFileService.class); + + @Autowired + private AdaptorSupport adaptorSupport; + + @Autowired + ThriftClientPool<RegistryService.Client> registryClientPool; + + private AgentAdaptor getAgentAdaptor(ProcessDataManager dataManager, String processId) throws Exception { + AgentAdaptor agentAdaptor; + try { + agentAdaptor = dataManager.getAgentAdaptor(); + } catch (Exception e) { + logger.error("Failed to fetch adaptor for process {}", processId, e); + throw new Exception("Failed to fetch adaptor for process " + processId, e); + } + return agentAdaptor; + } + public AiravataDirectory listFiles(String processId, String subPath) throws Exception { + ProcessDataManager dataManager = new ProcessDataManager(registryClientPool, processId, adaptorSupport); + + AgentAdaptor agentAdaptor = getAgentAdaptor(dataManager, processId); + + AiravataDirectory airavataDirectory = new AiravataDirectory("root", System.currentTimeMillis()); // TODO: set dir name + + subPath = dataManager.getBaseDir() + (subPath.isEmpty()? "" : "/" + subPath); + logger.info("Listing files in path {}", subPath); + List<String> fileList = agentAdaptor.listDirectory(subPath); // TODO: Validate if this is a file or dir + for (String fileOrDir : fileList) { + logger.info("Processing file {}", fileOrDir); + FileMetadata fileMetadata = agentAdaptor.getFileMetadata(subPath + "/" + fileOrDir); + airavataDirectory.getInnerFiles().add(new AiravataFile(fileMetadata.getName(), + fileMetadata.getSize(), System.currentTimeMillis(), System.currentTimeMillis())); // TODO: update created and updated time + } + + return airavataDirectory; + } + public Path downloadFile(String processId, String subPath) throws Exception { + + ProcessDataManager dataManager = new ProcessDataManager(registryClientPool, processId, adaptorSupport); + + AgentAdaptor agentAdaptor = getAgentAdaptor(dataManager, processId); + subPath = dataManager.getBaseDir() + (subPath.isEmpty()? "" : "/" + subPath); + + if (agentAdaptor.doesFileExist(subPath)) { + Path tempFile = Files.createTempFile("tempfile_", ".data"); + tempFile.toFile().deleteOnExit(); + try { + agentAdaptor.downloadFile(subPath, tempFile.toFile().getAbsolutePath()); + return tempFile; + } catch (Exception e) { + logger.error("Failed to download file {} from process {} to local path {}", + subPath, processId, tempFile.toFile().getAbsolutePath()); + try { + tempFile.toFile().delete(); + } catch (Exception ignore) { + // Ignore + } + throw e; + } + } else { + throw new Exception("File " + subPath + " does not exist in process " + processId); + } + } +} diff --git a/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java new file mode 100644 index 0000000000..b253ed11ee --- /dev/null +++ b/modules/file-server/src/main/java/org/apache/airavata/file/server/service/ProcessDataManager.java @@ -0,0 +1,57 @@ +package org.apache.airavata.file.server.service; + +import org.apache.airavata.agents.api.AgentAdaptor; +import org.apache.airavata.common.utils.ThriftClientPool; +import org.apache.airavata.helix.impl.task.TaskOnFailException; +import org.apache.airavata.helix.impl.task.staging.OutputDataStagingTask; +import org.apache.airavata.helix.task.api.support.AdaptorSupport; +import org.apache.airavata.model.experiment.ExperimentModel; +import org.apache.airavata.model.process.ProcessModel; +import org.apache.airavata.registry.api.RegistryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.UUID; + +public class ProcessDataManager extends OutputDataStagingTask { + + private final static Logger logger = LoggerFactory.getLogger(ProcessDataManager.class); + + private String processId; + private AdaptorSupport adaptorSupport; + + private ProcessModel process; + ExperimentModel experiment; + + public ProcessDataManager(ThriftClientPool<RegistryService.Client> registryClientPool, + String processId, AdaptorSupport adaptorSupport) throws Exception { + + this.adaptorSupport = adaptorSupport; + RegistryService.Client regClient = registryClientPool.getResource(); + try { + process = regClient.getProcess(processId); + experiment = regClient.getExperiment(process.getExperimentId()); + + setTaskId(UUID.randomUUID().toString()); + setProcessId(processId); + setExperimentId(process.getExperimentId()); + setGatewayId(experiment.getGatewayId()); + loadContext(); + + registryClientPool.returnResource(regClient); + } catch (Exception e) { + logger.error("Failed to initialize the output data mover for process {}", processId, e); + registryClientPool.returnBrokenResource(regClient); + throw e; + } + this.processId = processId; + } + + public AgentAdaptor getAgentAdaptor() throws TaskOnFailException { + return getComputeResourceAdaptor(adaptorSupport); + } + + public String getBaseDir() throws Exception { + return getTaskContext().getWorkingDir(); + } +} diff --git a/modules/file-server/src/main/resources/application.properties b/modules/file-server/src/main/resources/application.properties new file mode 100644 index 0000000000..3d5477b4dd --- /dev/null +++ b/modules/file-server/src/main/resources/application.properties @@ -0,0 +1,12 @@ +spring.servlet.multipart.max-file-size=10MB +# max request size +spring.servlet.multipart.max-request-size=10MB +# files storage location (stores all files uploaded via REST API) +storage.location=./uploads + +regserver.server.host=localhost +regserver.server.port=8970 +credential.store.server.host=localhost +credential.store.server.port=8960 + +server.port=8050 diff --git a/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh b/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh new file mode 100644 index 0000000000..131d99df54 --- /dev/null +++ b/modules/file-server/src/main/resources/distribution/bin/file-service-daemon.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +. `dirname $0`/setenv.sh +# Capture user's working dir before changing directory +CWD="$PWD" +cd ${AIRAVATA_HOME}/bin +LOGO_FILE="logo.txt" + +JAVA_OPTS=" -Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml" +AIRAVATA_COMMAND="" +EXTRA_ARGS="" +SERVERS="" +LOGO=true +IS_SUBSET=false +SUBSET="" +DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata-daemon.out" +LOG_FILE=$DEFAULT_LOG_FILE + +SERVICE_NAME="File Service" +PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid" + +case $1 in + start) + echo "Starting $SERVICE_NAME ..." + if [ ! -f $PID_PATH_NAME ]; then + nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \ + org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 & + echo $! > $PID_PATH_NAME + echo "$SERVICE_NAME started ..." + else + echo "$SERVICE_NAME is already running ..." + fi + ;; + stop) + if [ -f $PID_PATH_NAME ]; then + PID=$(cat $PID_PATH_NAME); + echo "$SERVICE_NAME stopping ..." + kill $PID; + RETRY=0 + while kill -0 $PID 2> /dev/null; do + echo "Waiting for the process $PID to be stopped" + RETRY=`expr ${RETRY} + 1` + if [ "${RETRY}" -gt "20" ] + then + echo "Forcefully killing the process as it is not responding ..." + kill -9 $PID + fi + sleep 1 + done + echo "$SERVICE_NAME stopped ..." + rm $PID_PATH_NAME + else + echo "$SERVICE_NAME is not running ..." + fi + ;; + restart) + if [ -f $PID_PATH_NAME ]; then + PID=$(cat $PID_PATH_NAME); + echo "$SERVICE_NAME stopping ..."; + kill $PID; + RETRY=0 + while kill -0 $PID 2> /dev/null; do + echo "Waiting for the process $PID to be stopped" + RETRY=`expr ${RETRY} + 1` + if [ "${RETRY}" -gt "20" ] + then + echo "Forcefully killing the process as it is not responding ..." + kill -9 $PID + fi + sleep 1 + done + echo "$SERVICE_NAME stopped ..."; + rm $PID_PATH_NAME + echo "$SERVICE_NAME starting ..." + nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \ + org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 & + echo $! > $PID_PATH_NAME + echo "$SERVICE_NAME started ..." + else + echo "$SERVICE_NAME is not running ..." + fi + ;; + -h) + echo "Usage: file-service-daemon.sh" + + echo "command options:" + echo " start Start server in daemon mode" + echo " stop Stop server running in daemon mode" + echo " restart Restart server in daemon mode" + echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)" + echo " -h Display this help and exit" + shift + exit 0 + ;; +esac diff --git a/modules/file-server/src/main/resources/distribution/bin/file-service.sh b/modules/file-server/src/main/resources/distribution/bin/file-service.sh new file mode 100644 index 0000000000..231f5dfa26 --- /dev/null +++ b/modules/file-server/src/main/resources/distribution/bin/file-service.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +. `dirname $0`/setenv.sh +# Capture user's working dir before changing directory +CWD="$PWD" +cd ${AIRAVATA_HOME}/bin +LOGO_FILE="logo.txt" + +JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlog4j.configurationFile=file:${AIRAVATA_HOME}/conf/log4j2.xml" +AIRAVATA_COMMAND="" +EXTRA_ARGS="" +SERVERS="" +IS_SUBSET=false +SUBSET="" +DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/airavata.out" +LOG_FILE=$DEFAULT_LOG_FILE + +# parse command arguments +for var in "$@" +do + case ${var} in + -xdebug) + AIRAVATA_COMMAND="${AIRAVATA_COMMAND}" + JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=*:8000" + shift + ;; + -log) + shift + LOG_FILE="$1" + shift + # If relative path, expand to absolute path using the user's $CWD + if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then + LOG_FILE="${CWD}/${LOG_FILE}" + fi + ;; + -h) + echo "Usage: file-service.sh" + + echo "command options:" + echo " -xdebug Start Connection Service under JPDA debugger" + echo " -h Display this help and exit" + shift + exit 0 + ;; + *) + EXTRA_ARGS="${EXTRA_ARGS} ${var}" + shift + ;; + esac +done + +java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \ + org.apache.airavata.file.server.FileServerApplication ${AIRAVATA_COMMAND} $* diff --git a/modules/file-server/src/main/resources/distribution/bin/setenv.sh b/modules/file-server/src/main/resources/distribution/bin/setenv.sh new file mode 100644 index 0000000000..3f1d7632bf --- /dev/null +++ b/modules/file-server/src/main/resources/distribution/bin/setenv.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# resolve links - $0 may be a softlink +PRG="$0" + +while [ -h "$PRG" ]; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`/"$link" + fi +done + +PRGDIR=`dirname "$PRG"` + +# Only set AIRAVATA_HOME if not already set +[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd` + +AIRAVATA_CLASSPATH="" + +for f in "$AIRAVATA_HOME"/lib/*.jar +do + AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f +done + +export AIRAVATA_HOME +export AIRAVATA_CLASSPATH \ No newline at end of file diff --git a/modules/file-server/src/main/resources/distribution/conf/application.properties b/modules/file-server/src/main/resources/distribution/conf/application.properties new file mode 100644 index 0000000000..3d5477b4dd --- /dev/null +++ b/modules/file-server/src/main/resources/distribution/conf/application.properties @@ -0,0 +1,12 @@ +spring.servlet.multipart.max-file-size=10MB +# max request size +spring.servlet.multipart.max-request-size=10MB +# files storage location (stores all files uploaded via REST API) +storage.location=./uploads + +regserver.server.host=localhost +regserver.server.port=8970 +credential.store.server.host=localhost +credential.store.server.port=8960 + +server.port=8050 diff --git a/modules/file-server/src/main/resources/distribution/conf/log4j2.xml b/modules/file-server/src/main/resources/distribution/conf/log4j2.xml new file mode 100644 index 0000000000..e38f7ff011 --- /dev/null +++ b/modules/file-server/src/main/resources/distribution/conf/log4j2.xml @@ -0,0 +1,53 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<Configuration status="WARN"> + + <Appenders> + <Console name="Console" target="SYSTEM_OUT"> + <PatternLayout pattern="%d [%t] %-5p %c{30} %X - %m%n"/> + </Console> + <RollingFile name="RollingFileAppender" fileName="../logs/airavata.log" + filePattern="logs/${date:yyyy-MM}/airavata-log-%d{MM-dd-yyyy}-%i.log.gz"> + <PatternLayout> + <Pattern>%d [%t] %-5p %c{30} %X - %m%n</Pattern> + </PatternLayout> + <Policies> + <OnStartupTriggeringPolicy /> + <TimeBasedTriggeringPolicy /> + <SizeBasedTriggeringPolicy size="50 MB" /> + </Policies> + <DefaultRolloverStrategy max="20" /> + </RollingFile> + </Appenders> + <Loggers> + <logger name="ch.qos.logback" level="WARN"/> + <logger name="org.apache.helix" level="WARN"/> + <logger name="org.apache.zookeeper" level="ERROR"/> + <logger name="org.apache.airavata" level="INFO"/> + <logger name="org.hibernate" level="ERROR"/> + <Root level="INFO"> + <AppenderRef ref="Console"/> + <AppenderRef ref="RollingFileAppender"/> + </Root> + </Loggers> +</Configuration> \ No newline at end of file diff --git a/pom.xml b/pom.xml index 3f57e2f519..f9ff579af0 100644 --- a/pom.xml +++ b/pom.xml @@ -102,6 +102,7 @@ <module>modules/agent-framework</module> <module>tools</module> <module>modules/ide-integration</module> + <module>modules/file-server</module> </modules> <properties> @@ -151,6 +152,7 @@ <org.apache.commons.pool2.version>2.7.0</org.apache.commons.pool2.version> <helix.version>0.9.7</helix.version> <httpclient.version>4.5.14</httpclient.version> + <spring.boot.version>3.2.4</spring.boot.version> </properties> <dependencies>
