This is an automated email from the ASF dual-hosted git repository. bossenti pushed a commit to branch allow-original-file-name in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit fbc3a3e8e420c2585059c6310d3193fe7988ded0 Author: Tim Bossenmaier <[email protected]> AuthorDate: Mon Oct 30 14:50:24 2023 +0100 feat: extend FileApi with the capability to query by the original file name --- .../apache/streampipes/client/api/IFileApi.java | 4 +- .../org/apache/streampipes/client/api/FileApi.java | 13 ++++-- .../adapters/image/ZipFileImageIterator.java | 2 +- .../jvm/processor/chunker/ChunkerProcessor.java | 2 +- .../language/LanguageDetectionProcessor.java | 2 +- .../processor/namefinder/NameFinderProcessor.java | 2 +- .../partofspeech/PartOfSpeechProcessor.java | 2 +- .../SentenceDetectionProcessor.java | 2 +- .../processor/tokenizer/TokenizerProcessor.java | 2 +- .../CsvMetadataEnrichmentProcessor.java | 2 +- .../streampipes/manager/file/FileManager.java | 12 ++++++ .../streampipes/rest/impl/PipelineElementFile.java | 50 ++++++++++++++++++++-- 12 files changed, 77 insertions(+), 18 deletions(-) diff --git a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java index 97d3da062..8238fcd90 100644 --- a/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java +++ b/streampipes-client-api/src/main/java/org/apache/streampipes/client/api/IFileApi.java @@ -19,9 +19,9 @@ package org.apache.streampipes.client.api; public interface IFileApi { - byte[] getFileContent(String filename); + byte[] getFileContent(String filename, boolean isOriginalFileName); - String getFileContentAsString(String filename); + String getFileContentAsString(String filename, boolean isOriginalFileName); void writeToFile(String file, String fileLocation); } diff --git a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java index 7d83d59f1..deae10828 100644 --- a/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java +++ b/streampipes-client/src/main/java/org/apache/streampipes/client/api/FileApi.java @@ -22,6 +22,9 @@ import org.apache.streampipes.client.http.BinaryGetRequest; import org.apache.streampipes.client.model.StreamPipesClientConfig; import org.apache.streampipes.client.util.StreamPipesApiPath; +import java.util.HashMap; +import java.util.Map; + public class FileApi extends AbstractClientApi implements IFileApi { @@ -30,13 +33,15 @@ public class FileApi extends AbstractClientApi implements IFileApi { } @Override - public byte[] getFileContent(String filename) { - return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename), null).executeRequest(); + public byte[] getFileContent(String filename, boolean isOriginalFileName) { + return new BinaryGetRequest(clientConfig, getBaseResourcePath(filename) + .withQueryParameters(Map.of("isOriginalFilename", String.valueOf(isOriginalFileName))), null) + .executeRequest(); } @Override - public String getFileContentAsString(String filename) { - return new String(getFileContent(filename)); + public String getFileContentAsString(String filename, boolean isOriginalFileName) { + return new String(getFileContent(filename, isOriginalFileName)); } @Override diff --git a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java index 73f1f6bab..79b0f8114 100644 --- a/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java +++ b/streampipes-extensions/streampipes-connect-adapters/src/main/java/org/apache/streampipes/connect/adapters/image/ZipFileImageIterator.java @@ -87,7 +87,7 @@ public class ZipFileImageIterator { private ZipInputStream fetchZipInputStream(String filename) { StreamPipesClient client = new StreamPipesClientResolver().makeStreamPipesClientInstance(); - byte[] result = client.fileApi().getFileContent(filename); + byte[] result = client.fileApi().getFileContent(filename, false); return new ZipInputStream(new ByteArrayInputStream(result)); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java index be8165468..7ab5818be 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/chunker/ChunkerProcessor.java @@ -98,7 +98,7 @@ public class ChunkerProcessor extends StreamPipesDataProcessor { this.tags = parameters.extractor().mappingPropertyValue(TAGS_FIELD_KEY); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename, false); InputStream modelIn = new ByteArrayInputStream(fileContent); ChunkerModel model; diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java index 862e61552..8210dd9e4 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/language/LanguageDetectionProcessor.java @@ -86,7 +86,7 @@ public class LanguageDetectionProcessor extends StreamPipesDataProcessor { SpOutputCollector spOutputCollector, EventProcessorRuntimeContext context) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = context.getStreamPipesClient().fileApi().getFileContent(filename,false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java index 0cbdc32d2..62abc061c 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/namefinder/NameFinderProcessor.java @@ -84,7 +84,7 @@ public class NameFinderProcessor extends StreamPipesDataProcessor { SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(MODEL); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.tokens = parameters.extractor().mappingPropertyValue(TOKENS_FIELD_KEY); loadModel(fileContent); } diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java index 5ef053c8f..eec1e6be0 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/partofspeech/PartOfSpeechProcessor.java @@ -86,7 +86,7 @@ public class PartOfSpeechProcessor extends StreamPipesDataProcessor { SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java index ae997b8cf..83ae73a1e 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/sentencedetection/SentenceDetectionProcessor.java @@ -73,7 +73,7 @@ public class SentenceDetectionProcessor extends StreamPipesDataProcessor { SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java index 2c68a393d..dc76845fe 100644 --- a/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java +++ b/streampipes-extensions/streampipes-processors-text-mining-jvm/src/main/java/org/apache/streampipes/processors/textmining/jvm/processor/tokenizer/TokenizerProcessor.java @@ -79,7 +79,7 @@ public class TokenizerProcessor extends StreamPipesDataProcessor { SpOutputCollector spOutputCollector, EventProcessorRuntimeContext runtimeContext) throws SpRuntimeException { String filename = parameters.extractor().selectedFilename(BINARY_FILE_KEY); - byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename); + byte[] fileContent = runtimeContext.getStreamPipesClient().fileApi().getFileContent(filename, false); this.detection = parameters.extractor().mappingPropertyValue(DETECTION_FIELD_KEY); InputStream modelIn = new ByteArrayInputStream(fileContent); diff --git a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java index 3bb61d265..7059d82ed 100644 --- a/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java +++ b/streampipes-extensions/streampipes-processors-transformation-jvm/src/main/java/org/apache/streampipes/processors/transformation/jvm/processor/csvmetadata/CsvMetadataEnrichmentProcessor.java @@ -186,7 +186,7 @@ public class CsvMetadataEnrichmentProcessor private String getFileContents(IParameterExtractor extractor) { String filename = extractor.selectedFilename(CSV_FILE_KEY); - return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename); + return getStreamPipesClientInstance().fileApi().getFileContentAsString(filename, false); } private StreamPipesClient getStreamPipesClientInstance() { diff --git a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java index 6ba0f85d1..ccde5c37d 100644 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/file/FileManager.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.stream.Collectors; @@ -43,6 +44,17 @@ public class FileManager { return filetypes != null ? filterFiletypes(allFiles, filetypes) : allFiles; } + public static File getFileByOriginalName(String originalName) throws IOException { + List<FileMetadata> allFiles = getFileMetadataStorage().getAllFileMetadataDescriptions(); + + var file = allFiles.stream().filter(fileMetadata -> fileMetadata.getOriginalFilename().equals(originalName)).findFirst(); + + if (file.isEmpty()){ + throw new IOException("No file with original name '%s' found".formatted(originalName)); + } + return new FileHandler().getFile(file.get().getInternalFilename()); + } + /** * Store a file in the internal file storage. * For csv files the bom is removed diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java index cf799a441..6e83f16e4 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java @@ -17,6 +17,12 @@ */ package org.apache.streampipes.rest.impl; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.enums.ParameterIn; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import jakarta.ws.rs.DefaultValue; +import org.apache.http.HttpStatus; import org.apache.streampipes.manager.file.FileManager; import org.apache.streampipes.model.file.FileMetadata; import org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource; @@ -24,6 +30,8 @@ import org.apache.streampipes.rest.security.AuthConstants; import org.glassfish.jersey.media.multipart.FormDataContentDisposition; import org.glassfish.jersey.media.multipart.FormDataParam; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.stereotype.Component; @@ -38,6 +46,8 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import javax.imageio.IIOException; +import java.io.IOException; import java.io.InputStream; @Path("/v2/files") @@ -76,8 +86,40 @@ public class PipelineElementFile extends AbstractAuthGuardedRestResource { @GET @Path("/{filename}") @Produces(MediaType.APPLICATION_OCTET_STREAM) - public Response getFile(@PathParam("filename") String filename) { - return ok(FileManager.getFile(filename)); - } - + @Operation( + summary = "Get file content by file name." + + "If multiple files with the same name exist, only the first is returned." + + "This can only be the case when the original file name is provided.", tags = {"Core", "Files"}, + responses = { + @ApiResponse( + responseCode = "" + HttpStatus.SC_OK, + description = "File could be found and is returned"), + @ApiResponse( + responseCode = "" + HttpStatus.SC_NOT_FOUND, + description = "No file with the given file name could be found") + } + ) + public Response getFile( + @Parameter( + in = ParameterIn.PATH, + description = "The name of the file to be retrieved", + required = true + ) + @PathParam("filename") String filename, + @Parameter( + in = ParameterIn.QUERY, + description = "Determines if the provided file name is the original file name" + + "as per upload." + ) + @QueryParam("isOriginalFilename") @DefaultValue("false") boolean isOriginalFilename + ) { + if (isOriginalFilename) { + try { + return ok(FileManager.getFileByOriginalName(filename)); + } catch (IOException e) { + return notFound(filename); + } + } + return ok(FileManager.getFile(filename)); + } }
