This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch 3448-add-excel-export-feature in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit d7b27ed033f280b67be400c50a02dc6645f50c5f Author: Dominik Riemer <[email protected]> AuthorDate: Thu Jan 30 22:38:22 2025 +0100 feat(#3448): Add excel export --- streampipes-data-explorer-export/pom.xml | 34 ++++-- .../export/ConfiguredCsvOutputWriter.java | 13 ++- .../export/ConfiguredExcelOutputWriter.java | 122 +++++++++++++++++++++ .../export/ConfiguredJsonOutputWriter.java | 6 +- .../export/ConfiguredOutputWriter.java | 37 ++++++- .../dataexplorer/export/OutputFormat.java | 14 ++- .../export/TestConfiguredCsvOutputWriter.java | 2 +- .../export/TestConfiguredJsonOutputWriter.java | 2 +- .../dataexplorer/StreamedQueryResultProvider.java | 5 +- .../datalake/param/SupportedRestQueryParams.java | 10 +- ...{PipelineElementFile.java => FileResource.java} | 4 +- .../rest/impl/datalake/DataLakeResource.java | 17 ++- .../src/lib/apis/datalake-rest.service.ts | 19 ++-- .../basic-inner-panel.component.html | 10 +- .../select-format/select-format.component.html | 78 ++++++++++++- .../select-format/select-format.component.ts | 27 ++++- .../data-download-dialog.component.ts | 3 +- .../model/format-export-config.model.ts | 18 ++- .../services/data-export.service.ts | 13 +-- .../services/file-name.service.spec.ts | 6 +- .../services/file-name.service.ts | 2 +- 21 files changed, 376 insertions(+), 66 deletions(-) diff --git a/streampipes-data-explorer-export/pom.xml b/streampipes-data-explorer-export/pom.xml index e108241836..0f7d6eac5d 100644 --- a/streampipes-data-explorer-export/pom.xml +++ b/streampipes-data-explorer-export/pom.xml @@ -29,6 +29,10 @@ This module contains all components and functionalities related to exporting data from the data explorer storage. </description> + <properties> + <apache-poi.version>5.4.0</apache-poi.version> + </properties> + <dependencies> <!-- StreamPipes dependencies --> <dependency> @@ -36,6 +40,27 @@ <artifactId>streampipes-model</artifactId> <version>0.98.0-SNAPSHOT</version> </dependency> + <dependency> + <groupId>org.apache.streampipes</groupId> + <artifactId>streampipes-storage-management</artifactId> + <version>0.98.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>org.apache.streampipes</groupId> + <artifactId>streampipes-pipeline-management</artifactId> + <version>0.98.0-SNAPSHOT</version> + </dependency> + + <dependency> + <groupId>org.apache.poi</groupId> + <artifactId>poi</artifactId> + <version>${apache-poi.version}</version> + </dependency> + <dependency> + <groupId>org.apache.poi</groupId> + <artifactId>poi-ooxml</artifactId> + <version>${apache-poi.version}</version> + </dependency> <!-- Test dependencies --> <dependency> @@ -44,11 +69,4 @@ <scope>test</scope> </dependency> </dependencies> - - <properties> - <maven.compiler.source>17</maven.compiler.source> - <maven.compiler.target>17</maven.compiler.target> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - </properties> - -</project> \ No newline at end of file +</project> diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java index 0f78498656..cbc7eaba95 100644 --- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java @@ -19,6 +19,7 @@ package org.apache.streampipes.dataexplorer.export; import org.apache.streampipes.dataexplorer.export.item.CsvItemGenerator; +import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; @@ -35,10 +36,18 @@ public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter { private CsvItemGenerator itemGenerator; private String delimiter = COMMA; + private DataLakeMeasure schema; + private String headerColumnNameStrategy; @Override - public void configure(ProvidedRestQueryParams params, + public void configure(DataLakeMeasure schema, + ProvidedRestQueryParams params, boolean ignoreMissingValues) { + this.schema = schema; + this.headerColumnNameStrategy = params + .getProvidedParams() + .getOrDefault(SupportedRestQueryParams.QP_HEADER_COLUMN_NAME, "key"); + if (params.has(SupportedRestQueryParams.QP_CSV_DELIMITER)) { delimiter = params.getAsString(SupportedRestQueryParams.QP_CSV_DELIMITER).equals("comma") ? COMMA : SEMICOLON; } @@ -69,7 +78,7 @@ public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter { private String makeHeaderLine(List<String> columns) { StringJoiner joiner = new StringJoiner(this.delimiter); - columns.forEach(joiner::add); + columns.forEach(c -> joiner.add(getHeaderName(schema, c, headerColumnNameStrategy))); return joiner + LINE_SEPARATOR; } } diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java new file mode 100644 index 0000000000..fbe5a86351 --- /dev/null +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.dataexplorer.export; + +import org.apache.streampipes.manager.file.FileManager; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams; +import org.apache.streampipes.model.file.FileMetadata; +import org.apache.streampipes.storage.api.CRUDStorage; + +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.List; +import java.util.Objects; + +public class ConfiguredExcelOutputWriter extends ConfiguredOutputWriter { + + private final CRUDStorage<FileMetadata> storage; + + private SXSSFWorkbook wb; + private Sheet ws; + private boolean useTemplate = false; + private int startRow = 0; + private String templateId; + private DataLakeMeasure schema; + private String headerColumnNameStrategy; + + + public ConfiguredExcelOutputWriter(CRUDStorage<FileMetadata> fileMetadataStorage) { + this.storage = fileMetadataStorage; + } + + @Override + public void configure(DataLakeMeasure schema, + ProvidedRestQueryParams params, + boolean ignoreMissingValues) { + var qp = params.getProvidedParams(); + this.schema = schema; + this.headerColumnNameStrategy = qp.getOrDefault(SupportedRestQueryParams.QP_HEADER_COLUMN_NAME, "key"); + if (qp.containsKey(SupportedRestQueryParams.QP_XLSX_USE_TEMPLATE)) { + this.useTemplate = true; + this.startRow = Integer.parseInt(qp.getOrDefault(SupportedRestQueryParams.QP_XLSX_START_ROW, "0")); + this.templateId = qp.getOrDefault(SupportedRestQueryParams.QP_XLSX_TEMPLATE_ID, null); + } + } + + @Override + public void beforeFirstItem(OutputStream outputStream) { + if (useTemplate && Objects.nonNull(templateId)) { + var fileMetadata = storage.getElementById(templateId); + if (fileMetadata != null) { + var path = new FileManager().getFile(fileMetadata.getFilename()).getAbsoluteFile().toPath(); + try (InputStream is = Files.newInputStream(path)) { + XSSFWorkbook templateWorkbook = new XSSFWorkbook(is); + wb = new SXSSFWorkbook(templateWorkbook); + ws = wb.getSheetAt(0); + } catch (IOException e) { + createNewWorkbook(); + } + } + } else { + createNewWorkbook(); + } + } + + private void createNewWorkbook() { + wb = new SXSSFWorkbook(); + ws = wb.createSheet(); + } + + @Override + public void afterLastItem(OutputStream outputStream) throws IOException { + wb.write(outputStream); + wb.close(); + } + + @Override + public void writeItem(OutputStream outputStream, + List<Object> row, + List<String> columnNames, boolean firstObject) throws IOException { + var excelRow = ws.createRow(startRow); + int columnIndex = 0; + if (!firstObject) { + for (Object column : row) { + var cell = excelRow.createCell(columnIndex); + cell.setCellValue(String.valueOf(column)); + columnIndex++; + } + } else { + for (String column : columnNames) { + var cell = excelRow.createCell(columnIndex); + var headerName = getHeaderName(schema, String.valueOf(column), headerColumnNameStrategy); + cell.setCellValue(headerName); + columnIndex++; + } + } + startRow++; + } +} diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java index d7d0b93c8c..5bfb631e02 100644 --- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java @@ -19,8 +19,9 @@ package org.apache.streampipes.dataexplorer.export; import org.apache.streampipes.dataexplorer.export.item.ItemGenerator; -import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import org.apache.streampipes.dataexplorer.export.item.JsonItemGenerator; +import org.apache.streampipes.model.datalake.DataLakeMeasure; +import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; import com.fasterxml.jackson.databind.ObjectMapper; @@ -40,7 +41,8 @@ public class ConfiguredJsonOutputWriter extends ConfiguredOutputWriter { } @Override - public void configure(ProvidedRestQueryParams params, + public void configure(DataLakeMeasure schema, + ProvidedRestQueryParams params, boolean ignoreMissingValues) { // do nothing } diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java index c7ff305843..6eac3b6145 100644 --- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java @@ -18,24 +18,55 @@ package org.apache.streampipes.dataexplorer.export; +import org.apache.streampipes.model.datalake.DataLakeMeasure; import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams; +import org.apache.streampipes.model.schema.EventProperty; import java.io.IOException; import java.io.OutputStream; import java.util.List; +import java.util.Objects; public abstract class ConfiguredOutputWriter { - public static ConfiguredOutputWriter getConfiguredWriter(OutputFormat format, + public static ConfiguredOutputWriter getConfiguredWriter(DataLakeMeasure schema, + OutputFormat format, ProvidedRestQueryParams params, boolean ignoreMissingValues) { var writer = format.getWriter(); - writer.configure(params, ignoreMissingValues); + writer.configure(schema, params, ignoreMissingValues); return writer; } - public abstract void configure(ProvidedRestQueryParams params, + protected String getHeaderName(DataLakeMeasure schema, + String runtimeName, + String headerColumnNameStrategy) { + if (Objects.nonNull(schema) && headerColumnNameStrategy.equals("label")) { + return schema + .getEventSchema() + .getEventProperties() + .stream() + .filter(ep -> ep.getRuntimeName().equals(runtimeName)) + .findFirst() + .map(ep -> extractLabel(ep, runtimeName)) + .orElse(runtimeName); + } else { + return runtimeName; + } + } + + private String extractLabel(EventProperty ep, + String runtimeName) { + if (Objects.nonNull(ep.getLabel())) { + return ep.getLabel(); + } else { + return runtimeName; + } + } + + public abstract void configure(DataLakeMeasure schema, + ProvidedRestQueryParams params, boolean ignoreMissingValues); public abstract void beforeFirstItem(OutputStream outputStream) throws IOException; diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java index 2342f5879f..d1237afc23 100644 --- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java +++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java @@ -18,11 +18,15 @@ package org.apache.streampipes.dataexplorer.export; +import org.apache.streampipes.storage.management.StorageDispatcher; + +import java.util.Arrays; import java.util.function.Supplier; public enum OutputFormat { JSON(ConfiguredJsonOutputWriter::new), - CSV(ConfiguredCsvOutputWriter::new); + CSV(ConfiguredCsvOutputWriter::new), + XLSX(() -> new ConfiguredExcelOutputWriter(StorageDispatcher.INSTANCE.getNoSqlStore().getFileMetadataStorage())); private final Supplier<ConfiguredOutputWriter> writerSupplier; @@ -33,4 +37,12 @@ public enum OutputFormat { public ConfiguredOutputWriter getWriter() { return writerSupplier.get(); } + + public static OutputFormat fromString(String desiredFormat) { + return Arrays.stream( + OutputFormat.values()) + .filter(format -> format.name().equalsIgnoreCase(desiredFormat)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(String.format("Could not find format %s", desiredFormat))); + } } diff --git a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java index 0fc7cec228..15a6b66e28 100644 --- a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java @@ -36,7 +36,7 @@ public class TestConfiguredCsvOutputWriter extends TestConfiguredOutputWriter { @Test public void testCsvOutputWriter() throws IOException { var writer = new ConfiguredCsvOutputWriter(); - writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true); + writer.configure(null, new ProvidedRestQueryParams(null, new HashMap<>()), true); try (var outputStream = new ByteArrayOutputStream()) { writer.beforeFirstItem(outputStream); diff --git a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java index b67b65ea28..3d5dc795d1 100644 --- a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java +++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java @@ -37,7 +37,7 @@ public class TestConfiguredJsonOutputWriter extends TestConfiguredOutputWriter { @Test public void testJsonOutputWriter() throws IOException { var writer = new ConfiguredJsonOutputWriter(); - writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true); + writer.configure(null, new ProvidedRestQueryParams(null, new HashMap<>()), true); try (var outputStream = new ByteArrayOutputStream()) { writer.beforeFirstItem(outputStream); diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java index 18e288ec98..936fc093d4 100644 --- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java +++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java @@ -51,15 +51,16 @@ public class StreamedQueryResultProvider extends QueryResultProvider { public void getDataAsStream(OutputStream outputStream) throws IOException { var usesLimit = queryParams.has(SupportedRestQueryParams.QP_LIMIT); + var measurement = findByMeasurementName(queryParams.getMeasurementId()).get(); + var configuredWriter = ConfiguredOutputWriter - .getConfiguredWriter(format, queryParams, ignoreMissingData); + .getConfiguredWriter(measurement, format, queryParams, ignoreMissingData); if (!queryParams.has(SupportedRestQueryParams.QP_LIMIT)) { queryParams.update(SupportedRestQueryParams.QP_LIMIT, MAX_RESULTS_PER_QUERY); } var limit = queryParams.getAsInt(SupportedRestQueryParams.QP_LIMIT); - var measurement = findByMeasurementName(queryParams.getMeasurementId()).get(); SpQueryResult dataResult; diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java index 881be1c659..4ec3a83894 100644 --- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java +++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java @@ -40,6 +40,10 @@ public class SupportedRestQueryParams { public static final String QP_AUTO_AGGREGATE = "autoAggregate"; public static final String QP_FILTER = "filter"; public static final String QP_MAXIMUM_AMOUNT_OF_EVENTS = "maximumAmountOfEvents"; + public static final String QP_XLSX_USE_TEMPLATE = "useTemplate"; + public static final String QP_XLSX_TEMPLATE_ID = "templateId"; + public static final String QP_XLSX_START_ROW = "startRow"; + public static final String QP_HEADER_COLUMN_NAME = "headerColumnName"; public static final List<String> SUPPORTED_PARAMS = Arrays.asList( QP_COLUMNS, @@ -58,7 +62,11 @@ public class SupportedRestQueryParams { QP_AUTO_AGGREGATE, QP_MISSING_VALUE_BEHAVIOUR, QP_FILTER, - QP_MAXIMUM_AMOUNT_OF_EVENTS + QP_MAXIMUM_AMOUNT_OF_EVENTS, + QP_XLSX_START_ROW, + QP_XLSX_TEMPLATE_ID, + QP_XLSX_USE_TEMPLATE, + QP_HEADER_COLUMN_NAME ); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java similarity index 98% rename from streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java rename to streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java index c5ff418e82..14b43b4eed 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java @@ -52,11 +52,11 @@ import static org.springframework.http.HttpStatus.NOT_FOUND; @RestController @RequestMapping("/api/v2/files") -public class PipelineElementFile extends AbstractAuthGuardedRestResource { +public class FileResource extends AbstractAuthGuardedRestResource { private final FileManager fileManager; - public PipelineElementFile() { + public FileResource() { this.fileManager = new FileManager(); } diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java index acc464d960..c757a61688 100644 --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java @@ -66,6 +66,7 @@ import static org.apache.streampipes.model.datalake.param.SupportedRestQueryPara import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FILTER; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FORMAT; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_GROUP_BY; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_HEADER_COLUMN_NAME; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_LIMIT; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MISSING_VALUE_BEHAVIOUR; @@ -74,6 +75,9 @@ import static org.apache.streampipes.model.datalake.param.SupportedRestQueryPara import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_PAGE; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_START_DATE; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_TIME_INTERVAL; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_START_ROW; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_TEMPLATE_ID; +import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_USE_TEMPLATE; import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.SUPPORTED_PARAMS; @RestController @@ -315,6 +319,14 @@ public class DataLakeResource extends AbstractRestResource { description = "filter conditions (a comma-separated list of filter conditions" + "such as [field,operator,condition])") @RequestParam(value = QP_FILTER, required = false) String filter, + @Parameter(in = ParameterIn.QUERY, description = "Excel export with template") + @RequestParam(value = QP_XLSX_USE_TEMPLATE, required = false) boolean useTemplate + , @Parameter(in = ParameterIn.QUERY, description = "ID of the excel template file to use") + @RequestParam(value = QP_XLSX_TEMPLATE_ID, required = false) String templateId + , @Parameter(in = ParameterIn.QUERY, description = "The first row in the excel file where data should be written") + @RequestParam(value = QP_XLSX_START_ROW, required = false) Integer startRow, + @Parameter(in = ParameterIn.QUERY, description = "Use either label or key as the column header") + @RequestParam(value = QP_HEADER_COLUMN_NAME, required = false) String headerColumnName, @RequestParam Map<String, String> queryParams) { @@ -326,7 +338,7 @@ public class DataLakeResource extends AbstractRestResource { format = "csv"; } - OutputFormat outputFormat = format.equals("csv") ? OutputFormat.CSV : OutputFormat.JSON; + var outputFormat = OutputFormat.fromString(format); StreamingResponseBody streamingOutput = output -> dataExplorerQueryManagement.getDataAsStream( sanitizedParams, outputFormat, @@ -335,7 +347,8 @@ public class DataLakeResource extends AbstractRestResource { HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_OCTET_STREAM); - headers.setContentDispositionFormData("attachment", "datalake." + outputFormat); + headers.setContentDispositionFormData("attachment", + "datalake." + outputFormat.toString().toLowerCase()); return ResponseEntity.ok() .headers(headers) diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts index 76f552ac3e..4b54c3e802 100644 --- a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts +++ b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts @@ -120,18 +120,16 @@ export class DatalakeRestService { downloadRawData( index: string, - format: string, - delimiter: string, + formatConfig: Record<string, any>, missingValueBehaviour: string, startTime?: number, endTime?: number, ) { const queryParams = startTime && endTime - ? { format, delimiter, startDate: startTime, endDate: endTime } + ? { ...formatConfig, startDate: startTime, endDate: endTime } : { - format, - delimiter, + ...formatConfig, missingValueBehaviour, }; return this.buildDownloadRequest(index, queryParams); @@ -139,23 +137,20 @@ export class DatalakeRestService { downloadQueriedData( index: string, - format: string, - delimiter: string, + formatConfig: Record<string, any>, missingValueBehaviour: string, queryParams: DatalakeQueryParameters, ) { - (queryParams as any).format = format; - (queryParams as any).delimiter = delimiter; - (queryParams as any).missingValueBehaviour = missingValueBehaviour; + const qp = { ...formatConfig, ...queryParams, missingValueBehaviour }; - return this.buildDownloadRequest(index, queryParams); + return this.buildDownloadRequest(index, qp); } buildDownloadRequest(index: string, queryParams: any) { const url = this.dataLakeUrl + '/measurements/' + index + '/download'; const request = new HttpRequest('GET', url, { reportProgress: true, - responseType: 'text', + responseType: 'blob', params: this.toHttpParams(queryParams), }); diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html b/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html index 73038461b7..7cae7a6df9 100644 --- a/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html +++ b/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html @@ -20,12 +20,11 @@ fxLayout="column" fxFlex="100" class="panel-outer" - [ngStyle]="{ margin: outerMargin }" + [ngStyle]="{ margin: outerMargin, height: '100%' }" > <div class="general-panel-header" fxLayout="row" - fxFlex="100" fxLayoutAlign="start center" *ngIf="!hideToolbar" > @@ -35,11 +34,14 @@ {{ panelTitle }} </div> </div> - <ng-content select="[header]" fxFlex class="pr-5"></ng-content> + <ng-content select="[header]" class="pr-5"></ng-content> </div> </div> - <div class="general-panel" [ngStyle]="{ padding: innerPadding }"> + <div + class="general-panel" + [ngStyle]="{ padding: innerPadding, height: '100%' }" + > <ng-content fxFlex="100"></ng-content> </div> </div> diff --git a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html index 77a4f7445b..423ac7035c 100644 --- a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html +++ b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html @@ -21,7 +21,7 @@ <h5>Download Format</h5> <mat-radio-group class="sp-radio-group" - [(ngModel)]="formatExportConfig.exportFormat" + [(ngModel)]="formatExportConfig.format" > <mat-radio-button value="json" @@ -37,9 +37,16 @@ > CSV </mat-radio-button> + <mat-radio-button + value="xlsx" + class="sp-radio-button" + data-cy="download-configuration-xlsx" + > + Excel + </mat-radio-button> </mat-radio-group> </div> -<div *ngIf="formatExportConfig.exportFormat === 'csv'" class="mt-10"> +<div *ngIf="formatExportConfig.format === 'csv'" class="mt-10"> <h5>Delimiter</h5> <mat-radio-group [(ngModel)]="formatExportConfig.delimiter" @@ -49,13 +56,72 @@ value="comma" class="sp-radio-button" data-cy="download-configuration-delimiter-comma" - > ,</mat-radio-button - > + > , + </mat-radio-button> <mat-radio-button value="semicolon" class="sp-radio-button" data-cy="download-configuration-delimiter-semicolon" - > ;</mat-radio-button - > + > ; + </mat-radio-button> + </mat-radio-group> +</div> +<div + *ngIf="formatExportConfig.format === 'xlsx' && hasReadFilePrivilege" + fxLayout="column" +> + <mat-checkbox + [(ngModel)]="formatExportConfig.useTemplate" + [disabled]="excelTemplates.length === 0" + > + Use uploaded file template + </mat-checkbox> + @if (formatExportConfig.useTemplate && excelTemplates.length > 0) { + <mat-form-field class="mt-10" color="accent"> + <mat-select + [(ngModel)]="formatExportConfig.templateId" + placeholder="Choose template" + > + <mat-option + *ngFor="let template of excelTemplates" + [value]="template.fileId" + > + {{ template.filename }} + </mat-option> + </mat-select> + </mat-form-field> + <mat-form-field color="accent"> + <mat-label>First row index to append data</mat-label> + <input + matInput + [(ngModel)]="formatExportConfig.startRow" + type="number" + /> + </mat-form-field> + } +</div> +<div + *ngIf=" + formatExportConfig.format === 'xlsx' || + formatExportConfig.format === 'csv' + " +> + <h5>Header column name</h5> + <mat-radio-group + [(ngModel)]="formatExportConfig.headerColumnName" + class="sp-radio-group" + > + <mat-radio-button + value="key" + class="sp-radio-button" + data-cy="download-configuration-column-name-key" + >Use field key (runtime name) as header column + </mat-radio-button> + <mat-radio-button + value="label" + class="sp-radio-button" + data-cy="download-configuration-column-name-label" + >Use field label as header column if available + </mat-radio-button> </mat-radio-group> </div> diff --git a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts index d69b5e267c..23db806489 100644 --- a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts +++ b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts @@ -16,8 +16,11 @@ * */ -import { Component, Input } from '@angular/core'; +import { Component, inject, Input, OnInit } from '@angular/core'; import { FormatExportConfig } from '../../model/format-export-config.model'; +import { FileMetadata, FilesService } from '@streampipes/platform-services'; +import { UserPrivilege } from '../../../../_enums/user-privilege.enum'; +import { CurrentUserService } from '@streampipes/shared-ui'; @Component({ selector: 'sp-select-format', @@ -27,8 +30,28 @@ import { FormatExportConfig } from '../../model/format-export-config.model'; '../../data-download-dialog.component.scss', ], }) -export class SelectFormatComponent { +export class SelectFormatComponent implements OnInit { @Input() formatExportConfig: FormatExportConfig; + hasReadFilePrivilege = false; + excelTemplates: FileMetadata[] = []; + + private fileService = inject(FilesService); + private currentUserService = inject(CurrentUserService); + constructor() {} + + ngOnInit() { + this.hasReadFilePrivilege = this.currentUserService.hasRole( + UserPrivilege.PRIVILEGE_READ_FILES, + ); + if (this.hasReadFilePrivilege) { + this.fileService + .getFileMetadata(['xlsx']) + .subscribe(excelTemplates => { + this.excelTemplates = excelTemplates; + console.log(this.excelTemplates); + }); + } + } } diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts index 4a0b6a199b..59e7ea58cd 100644 --- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts +++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts @@ -55,8 +55,9 @@ export class DataDownloadDialogComponent implements OnInit { measurement: measurementName, }, formatExportConfig: { - exportFormat: 'csv', + format: 'csv', delimiter: 'comma', + headerColumnName: 'key', }, }; } diff --git a/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts b/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts index 6562bfb909..fab91f13c0 100644 --- a/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts +++ b/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts @@ -34,13 +34,25 @@ * */ -export type FormatExportConfig = JsonFormatExportConfig | CsvFormatExportConfig; +export type FormatExportConfig = + | JsonFormatExportConfig + | CsvFormatExportConfig + | ExcelFormatConfig; export interface JsonFormatExportConfig { - exportFormat: 'json'; + format: 'json'; } export interface CsvFormatExportConfig { - exportFormat: 'csv'; + format: 'csv'; delimiter: 'comma' | 'semicolon'; + headerColumnName: 'key' | 'label'; +} + +export interface ExcelFormatConfig { + format: 'xlsx'; + templateId: string; + startRow: number; + useTemplate: boolean; + headerColumnName: 'key' | 'label'; } diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts index 4725487714..a014ed97f9 100644 --- a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts +++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts @@ -52,8 +52,7 @@ export class DataExportService { ) { downloadRequest = this.dataLakeRestService.downloadQueriedData( exportConfig.dataExportConfig.measurement, - exportConfig.formatExportConfig.exportFormat, - exportConfig.formatExportConfig['delimiter'], + exportConfig.formatExportConfig, exportConfig.dataExportConfig.missingValueBehaviour, this.generateQueryRequest( exportConfig, @@ -75,8 +74,7 @@ export class DataExportService { } downloadRequest = this.dataLakeRestService.downloadRawData( exportConfig.dataExportConfig.measurement, - exportConfig.formatExportConfig.exportFormat, - exportConfig.formatExportConfig['delimiter'], + exportConfig.formatExportConfig, exportConfig.dataExportConfig.missingValueBehaviour, startTime, endTime, @@ -139,12 +137,9 @@ export class DataExportService { exportConfig, new Date(), ); + const blob = new Blob([data]); - const url = window.URL.createObjectURL( - new Blob([String(data)], { - type: `data:text/${exportConfig.formatExportConfig.exportFormat};charset=utf-8`, - }), - ); + const url = window.URL.createObjectURL(blob); a.href = url; a.download = name; a.click(); diff --git a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts index 74430dc99e..deef6c48ed 100644 --- a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts +++ b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts @@ -40,7 +40,7 @@ describe('FileNameService', () => { measurement: 'measurement', }, formatExportConfig: { - exportFormat: 'csv', + format: 'csv', delimiter: 'comma', }, }; @@ -55,7 +55,7 @@ describe('FileNameService', () => { }); it('Name for all data json', () => { - defaultExportConfig.formatExportConfig.exportFormat = 'json'; + defaultExportConfig.formatExportConfig.format = 'json'; const result = service.generateName( defaultExportConfig, defaultExportDate, @@ -78,7 +78,7 @@ describe('FileNameService', () => { }); it('Name for custom visible json', () => { - defaultExportConfig.formatExportConfig.exportFormat = 'json'; + defaultExportConfig.formatExportConfig.format = 'json'; defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible'; defaultExportConfig.dataExportConfig.dateRange = defaultDateRange; diff --git a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts index 878bd50355..66d8fe5b60 100644 --- a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts +++ b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts @@ -30,7 +30,7 @@ export class FileNameService { const dataRangeOption = exportConfig.dataExportConfig.dataRangeConfiguration; let dateRange = ''; - const fileExtension = `.${exportConfig.formatExportConfig.exportFormat}`; + const fileExtension = `.${exportConfig.formatExportConfig.format}`; if ( exportConfig.dataExportConfig.dateRange !== undefined &&
