This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch fork/dev in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit f6758b69f0ad7b93192b3ae80f349087f14d757e Merge: 85789e186 6d7efad88 Author: Philipp Zehnder <[email protected]> AuthorDate: Wed Dec 20 18:09:59 2023 +0100 Merge branch 'dev' into fork/dev .github/pr-labeler-config.yml | 85 +++-- .github/workflows/pr-labeler.yml | 2 +- .github/workflows/pr-validation.yml | 6 +- .github/workflows/pypi-deployment.yml | 2 +- .github/workflows/python-docs.yml | 2 +- .github/workflows/stale-bot.yml | 2 +- .../src/main/resources/archetype-resources/pom.xml | 5 - .../src/main/resources/archetype-resources/pom.xml | 5 - installer/cli/.env | 2 +- installer/cli/README.md | 2 +- installer/cli/environments/lite-monitoring | 1 - installer/compose/.env | 2 +- installer/compose/README.md | 2 +- installer/k8s/Chart.yaml | 2 +- installer/k8s/README.md | 4 +- pom.xml | 57 ++-- streampipes-client-python/poetry.lock | 213 ++++++++++-- streampipes-client-python/pyproject.toml | 4 +- .../commons/constants/HttpConstants.java | 1 + .../prometheus/StreamPipesCollectorRegistry.java | 8 + .../commons/prometheus/adapter/AdapterMetrics.java | 126 +++++++ .../adapter/AdapterMetricsManager.java} | 28 +- .../streampipes/commons/resources/Resources.java | 20 +- .../prometheus/adapter/AdapterMetricsTest.java | 65 ++++ .../commons/resources}/TestResources.java | 4 +- .../management/health/AdapterHealthCheck.java | 198 ++++++++--- .../management/AdapterMasterManagement.java | 30 +- .../management/WorkerAdministrationManagement.java | 20 +- .../management/health/AdapterHealthCheckTest.java | 97 ++++++ .../management/AdapterMasterManagementTest.java | 29 +- .../dataexplorer/commons/TimeSeriesStore.java | 6 - .../commons/influx/InfluxClientProvider.java | 2 + streampipes-data-explorer/pom.xml | 15 +- .../dataexplorer/DataExplorerSchemaManagement.java | 203 +++++++---- .../api/IDataExplorerSchemaManagement.java | 2 +- .../influx/DataExplorerInfluxQueryExecutor.java | 31 +- .../param/ProvidedRestQueryParamConverter.java | 2 - .../dataexplorer/param/SelectQueryParams.java | 6 - .../dataexplorer/query/AutoAggregationHandler.java | 35 +- .../query/DataExplorerQueryExecutor.java | 50 +-- .../DataExplorerSchemaManagementTest.java | 204 ++++++++++++ streampipes-extensions/pom.xml | 2 +- .../documentation.md | 17 +- .../connectors/influx/sink/InfluxDbClient.java | 17 +- .../streampipes-extensions-all-iiot/Dockerfile | 2 +- .../streampipes-extensions-all-jvm/Dockerfile | 2 +- .../streampipes-extensions-iiot-minimal/Dockerfile | 2 +- .../processor/compose/TestComposeProcessor.java | 166 +++++++++ .../TestSwingingDoorTrendingFilterProcessor.java | 3 +- .../jvm/TransformationExtensionModuleExport.java | 4 +- .../datetime/DateTimeFromStringProcessor.java | 153 +++++++++ .../documentation.md | 72 ++++ .../icon.png | Bin 0 -> 259252 bytes .../strings.en | 14 +- .../datetime/TestDateTimeFromStringProcessor.java | 250 ++++++++++++++ .../streampipes-sinks-internal-jvm/pom.xml | 11 + .../jvm/InternalSinksExtensionModuleExports.java | 7 +- .../sinks/internal/jvm/datalake/DataLakeSink.java | 56 +++- .../migrations/DataLakeSinkMigrationV1.java | 79 +++++ ...va => InternalStreamPipesNotificationSink.java} | 84 ++--- .../documentation.md | 22 +- .../strings.en | 3 + .../documentation.md | 6 + .../strings.en | 3 - .../migrations/DataLakeSinkMigrationV1Test.java | 62 ++++ .../sinks/notifications/jvm/email/EmailSink.java | 51 +-- .../notifications/jvm/msteams/MSTeamsSink.java | 131 ++++---- .../notifications/jvm/onesignal/OneSignalSink.java | 6 +- .../jvm/slack/SlackNotificationSink.java | 14 +- .../notifications/jvm/telegram/TelegramSink.java | 6 +- .../documentation.md | 8 +- .../strings.en | 3 - .../documentation.md | 6 + .../documentation.md | 1 - streampipes-integration-tests/pom.xml | 2 +- streampipes-maven-plugin/pom.xml | 6 +- streampipes-messaging-pulsar/pom.xml | 2 +- .../streampipes/model/client/user/Group.java | 14 +- .../apache/streampipes/model/client/user/Role.java | 5 +- .../model/datalake/DataLakeMeasure.java | 10 + .../DataLakeMeasureSchemaUpdateStrategy.java | 25 +- .../streampipes/model/message/Notification.java | 6 +- .../model/migration/MigrationResult.java | 8 +- .../migration/AbstractMigrationManager.java | 3 +- .../streampipes/ps/DataLakeImageResource.java | 22 +- .../streampipes/ps/DataLakeMeasureResourceV4.java | 76 ++--- .../apache/streampipes/ps/DataLakeResourceV3.java | 21 +- .../apache/streampipes/ps/DataLakeResourceV4.java | 227 +++++++------ .../ps/PipelineElementTemplateResource.java | 120 +++---- .../base/impl/AbstractAuthGuardedRestResource.java | 21 +- .../rest/core/base/impl/AbstractRestResource.java | 27 +- streampipes-rest-extensions/pom.xml | 4 - .../extensions/AbstractExtensionsResource.java | 35 +- .../AbstractPipelineElementResource.java | 42 +-- .../streampipes/rest/extensions/WelcomePage.java | 17 +- .../extensions/connect/AdapterAssetResource.java | 44 ++- .../connect/AdapterDescriptionResource.java | 32 +- .../extensions/connect/AdapterWorkerResource.java | 60 ++-- .../rest/extensions/connect/GuessResource.java | 33 +- .../connect/HttpServerAdapterResource.java | 28 +- .../connect/RuntimeResolvableResource.java | 41 ++- .../migration/AdapterMigrationResource.java | 32 +- .../migration/DataProcessorMigrationResource.java | 29 +- .../migration/DataSinkMigrationResource.java | 75 +++-- .../extensions/monitoring/MonitoringResource.java | 19 +- .../pe/DataProcessorPipelineElementResource.java | 6 +- .../pe/DataSinkPipelineElementResource.java | 6 +- .../pe/DataStreamPipelineElementResource.java | 21 +- .../pe/InvocablePipelineElementResource.java | 80 ++--- streampipes-rest-shared/pom.xml | 41 +-- .../annotation/NoAuthenticationRequired.java | 28 -- .../streampipes/rest/shared/api/CRUDResource.java | 33 -- .../exception/RestResponseExceptionHandler.java | 38 +++ .../RestResponseLogMessageExceptionHandler.java | 28 +- .../RestResponseMessageExceptionHandler.java | 39 +++ .../shared/exception/SpLogMessageException.java | 28 +- .../rest/shared/exception/SpMessageException.java | 38 ++- .../shared/exception/SpNotificationException.java | 29 +- .../shared/impl/AbstractSharedRestInterface.java | 51 ++- .../serializer/JacksonSerializationProvider.java | 64 ---- .../rest/shared/serializer/JsonJerseyProvider.java | 116 ------- .../streampipes/rest/shared/util/SpMediaType.java | 30 -- .../apache/streampipes/rest/ResetManagement.java | 18 +- .../rest/impl/AbstractMonitoringResource.java | 8 +- .../rest/impl/AccountActivationResource.java | 30 +- .../rest/impl/AdapterMonitoringResource.java | 31 +- .../rest/impl/AssetDashboardResource.java | 101 +++--- .../rest/impl/AssetManagementResource.java | 72 ++-- .../streampipes/rest/impl/Authentication.java | 84 +++-- .../apache/streampipes/rest/impl/AutoComplete.java | 23 +- .../streampipes/rest/impl/CategoryResource.java | 67 ++-- .../rest/impl/ContainerProvidedOptions.java | 27 +- .../apache/streampipes/rest/impl/DataStream.java | 22 +- .../streampipes/rest/impl/EmailResource.java | 20 +- .../streampipes/rest/impl/FunctionsResource.java | 63 ++-- .../rest/impl/GenericStorageResource.java | 88 +++-- .../streampipes/rest/impl/LabelResource.java | 73 ++-- .../rest/impl/MeasurementUnitResource.java | 33 +- .../apache/streampipes/rest/impl/Notification.java | 110 ------ .../rest/impl/NotificationResource.java | 99 ++++++ .../rest/impl/OntologyMeasurementUnit.java | 38 +-- .../streampipes/rest/impl/PipelineCache.java | 33 +- .../rest/impl/PipelineCanvasMetadataCache.java | 33 +- .../rest/impl/PipelineCanvasMetadataResource.java | 84 +++-- .../streampipes/rest/impl/PipelineCategory.java | 49 ++- .../rest/impl/PipelineElementAsset.java | 34 +- .../rest/impl/PipelineElementCategory.java | 46 +-- .../streampipes/rest/impl/PipelineElementFile.java | 107 +++--- .../rest/impl/PipelineElementPreview.java | 53 ++- .../rest/impl/PipelineElementRuntimeInfo.java | 26 +- .../streampipes/rest/impl/PipelineMonitoring.java | 34 +- .../streampipes/rest/impl/PipelineResource.java | 172 +++++----- .../streampipes/rest/impl/PipelineTemplate.java | 77 +++-- .../streampipes/rest/impl/ResetResource.java | 25 +- .../rest/impl/RestorePasswordResource.java | 37 +-- .../org/apache/streampipes/rest/impl/Setup.java | 22 +- .../apache/streampipes/rest/impl/UserResource.java | 145 ++++---- .../org/apache/streampipes/rest/impl/Version.java | 29 +- .../rest/impl/admin/DataExportResource.java | 38 +-- .../rest/impl/admin/DataImportResource.java | 48 ++- .../impl/admin/EmailConfigurationResource.java | 60 ++-- .../admin/ExtensionsServiceEndpointResource.java | 69 ++-- .../impl/admin/GeneralConfigurationResource.java | 61 ++-- .../impl/admin/MessagingConfigurationResource.java | 37 +-- .../rest/impl/admin/MigrationResource.java | 34 +- .../rest/impl/admin/PermissionResource.java | 29 +- .../rest/impl/admin/PipelineElementImport.java | 58 ++-- .../impl/admin/ServiceConfigurationResource.java | 58 ++-- .../impl/admin/ServiceRegistrationResource.java | 44 +-- .../rest/impl/admin/UserAdminResource.java | 26 +- .../rest/impl/admin/UserGroupResource.java | 50 +-- .../rest/impl/connect/AdapterResource.java | 112 +++---- .../rest/impl/connect/DescriptionResource.java | 51 ++- .../rest/impl/connect/GuessResource.java | 39 +-- .../impl/connect/RuntimeResolvableResource.java | 44 +-- .../rest/impl/connect/UnitResource.java | 26 +- .../impl/dashboard/AbstractDashboardResource.java | 48 +-- .../AbstractPipelineExtractionResource.java | 8 +- .../streampipes/rest/impl/dashboard/Dashboard.java | 9 +- .../rest/impl/dashboard/DashboardWidget.java | 68 ++-- .../dashboard/VisualizablePipelineResource.java | 34 +- .../impl/datalake/DataLakeDashboardResource.java | 9 +- .../rest/impl/datalake/DataLakeWidgetResource.java | 69 ++-- .../impl/datalake/PersistedDataStreamResource.java | 34 +- .../rest/impl/pe/DataProcessorResource.java | 45 +-- .../streampipes/rest/impl/pe/DataSinkResource.java | 46 +-- .../rest/impl/pe/DataStreamResource.java | 59 ++-- .../sdk/builder/NestedPropertyBuilder.java | 94 ++++++ .../sdk/builder/NestedPropertyBuilderTest.java | 103 ++++++ streampipes-service-base/pom.xml | 18 - .../service/base/rest/ServiceHealthResource.java | 20 +- .../security/UnauthorizedRequestEntryPoint.java | 2 +- streampipes-service-core-minimal/Dockerfile | 2 +- .../minimal/StreamPipesCoreApplicationMinimal.java | 13 +- streampipes-service-core/Dockerfile | 2 +- streampipes-service-core/pom.xml | 33 +- .../service/core/OpenApiConfiguration.java | 74 +++++ .../streampipes/service/core/PostStartupTask.java | 34 +- .../service/core/StreamPipesCoreApplication.java | 25 +- .../service/core/StreamPipesResourceConfig.java | 191 ----------- .../service/core/UnauthenticatedInterfaces.java | 2 +- .../service/core/WebSecurityConfig.java | 6 + .../core/filter/TokenAuthenticationFilter.java | 50 ++- .../src/main/resources/application.properties | 4 + .../src/main/resources/openapi.yaml | 35 -- .../extensions/ExtensionsModelSubmitter.java | 6 +- .../extensions/ExtensionsResourceConfig.java | 76 ----- streampipes-user-management/pom.xml | 5 - .../java/org/apache/streampipes/vocabulary/SO.java | 1 + .../wrapper/standalone/StreamPipesDataSink.java | 15 +- .../standalone/StreamPipesNotificationSink.java | 136 ++++++++ .../support/utils/DataDownloadDialogUtils.ts | 10 +- ui/cypress/support/utils/UserUtils.ts | 1 + .../tests/dashboard/editDashboardTest.spec.ts | 4 +- ui/package-lock.json | 44 +-- ui/package.json | 6 +- .../src/lib/model/gen/streampipes-model-client.ts | 2 +- .../src/lib/model/gen/streampipes-model.ts | 14 +- ui/src/app/apidocs/apidocs.component.ts | 2 +- .../abstract-security-principal-config.ts | 28 +- .../user-group-configuration.component.ts | 26 +- .../adapter-started-dialog.component.ts | 4 + ui/src/app/core-model/coco/Annotation.ts | 41 --- ui/src/app/core-model/coco/Category.ts | 36 -- ui/src/app/core-model/coco/Coco.format.ts | 38 --- ui/src/app/core-model/coco/Image.ts | 28 -- .../template/PipelineInvocationBuilder.ts | 20 ++ ui/src/app/core-ui/core-ui.module.ts | 32 -- .../image-annotations.component.css | 33 -- .../image-annotations.component.html | 51 --- .../image-annotations.component.ts | 56 ---- .../image-container/image-container.component.css | 23 -- .../image-container/image-container.component.html | 31 -- .../image-container/image-container.component.ts | 70 ---- .../image-labeling/image-labeling.component.css | 17 - .../image-labeling/image-labeling.component.html | 151 --------- .../image-labeling/image-labeling.component.ts | 370 --------------------- ui/src/app/core-ui/image/image.component.css | 17 - ui/src/app/core-ui/image/image.component.html | 32 -- ui/src/app/core-ui/image/image.component.ts | 32 -- ui/src/app/core-ui/image/model/coordinates.ts | 21 -- ui/src/app/core-ui/image/model/labeling-mode.ts | 23 -- .../image/services/BrushLabeling.service.ts | 181 ---------- .../core-ui/image/services/CocoFormat.service.ts | 169 ---------- .../core-ui/image/services/LabelingMode.service.ts | 77 ----- .../image/services/PolygonLabeling.service.ts | 316 ------------------ .../image/services/ReactLabeling.service.ts | 245 -------------- ui/src/app/core-ui/image/services/color.service.ts | 36 -- ui/src/app/core-ui/image/util/color.util.ts | 33 -- .../core-ui/image/util/imageTranslation.util.ts | 38 --- .../classification/imageClassification.ts | 50 --- .../configure-labels/configure-labels.component.ts | 6 +- .../select-label/select-label.component.ts | 6 +- .../image-bar-preview.component.html | 0 .../image-bar-preview.component.scss | 0 .../image-bar-preview.component.ts | 0 .../image}/image-bar/image-bar.component.css | 0 .../image}/image-bar/image-bar.component.html | 0 .../image}/image-bar/image-bar.component.ts | 0 .../image/image-viewer/image-viewer.component.css | 0 .../image/image-viewer/image-viewer.component.html | 0 .../image/image-viewer/image-viewer.component.ts | 0 ui/src/app/data-explorer/data-explorer.module.ts | 6 + .../save-pipeline/save-pipeline.component.ts | 1 - ui/src/app/editor/services/jsplumb.service.ts | 1 - .../components/auth-box/auth-box.component.html | 90 +++-- .../components/auth-box/auth-box.component.scss | 50 +++ .../components/auth-box/auth-box.component.ts | 5 +- 268 files changed, 5397 insertions(+), 6054 deletions(-) diff --cc streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java index c6fdf482b,72a3f4d0a..fbc27a87b --- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java +++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java @@@ -27,39 -29,39 +29,41 @@@ import io.swagger.v3.oas.annotations.Pa import io.swagger.v3.oas.annotations.enums.ParameterIn; import io.swagger.v3.oas.annotations.responses.ApiResponse; import org.apache.http.HttpStatus; - import org.glassfish.jersey.media.multipart.FormDataContentDisposition; - import org.glassfish.jersey.media.multipart.FormDataParam; - import org.springframework.security.access.prepost.PreAuthorize; - import org.springframework.stereotype.Component; + - import jakarta.ws.rs.Consumes; - import jakarta.ws.rs.DELETE; - import jakarta.ws.rs.DefaultValue; - import jakarta.ws.rs.GET; - import jakarta.ws.rs.POST; - import jakarta.ws.rs.Path; - import jakarta.ws.rs.PathParam; - import jakarta.ws.rs.Produces; - import jakarta.ws.rs.QueryParam; - import jakarta.ws.rs.core.MediaType; - import jakarta.ws.rs.core.Response; + import org.springframework.http.MediaType; + import org.springframework.http.ResponseEntity; + import org.springframework.security.access.prepost.PreAuthorize; + import org.springframework.web.bind.annotation.DeleteMapping; + import org.springframework.web.bind.annotation.GetMapping; + import org.springframework.web.bind.annotation.PathVariable; + import org.springframework.web.bind.annotation.PostMapping; + import org.springframework.web.bind.annotation.RequestMapping; + import org.springframework.web.bind.annotation.RequestParam; + import org.springframework.web.bind.annotation.RequestPart; + import org.springframework.web.bind.annotation.RestController; + import org.springframework.web.multipart.MultipartFile; + import java.io.File; import java.io.IOException; - import java.io.InputStream; - import java.util.stream.Collectors; + import java.nio.file.Files; + import java.util.List; - @Path("/v2/files") - @Component + @RestController + @RequestMapping("/api/v2/files") public class PipelineElementFile extends AbstractAuthGuardedRestResource { - @POST - @Consumes(MediaType.MULTIPART_FORM_DATA) + @PostMapping( + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize(AuthConstants.IS_ADMIN_ROLE) - public Response storeFile(@FormDataParam("file_upload") InputStream uploadedInputStream, - @FormDataParam("file_upload") FormDataContentDisposition fileDetail) { + public ResponseEntity<?> storeFile(@RequestPart("file_upload") MultipartFile fileDetail) { try { FileMetadata metadata = - FileManager.storeFile(getAuthenticatedUsername(), fileDetail.getFileName(), uploadedInputStream); + FileManager.storeFile( + getAuthenticatedUsername(), + fileDetail.getOriginalFilename(), - fileDetail.getInputStream()); ++ fileDetail.getInputStream() ++ ); return ok(metadata); } catch (Exception e) { return fail(); @@@ -74,10 -75,10 +77,11 @@@ return ok(); } - @GET - @Produces(MediaType.APPLICATION_JSON) + @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize(AuthConstants.HAS_READ_FILE_PRIVILEGE) - public Response getFileInfo(@QueryParam("filetypes") String filetypes) { + public ResponseEntity<List<FileMetadata>> getFileInfo( - @RequestParam(value = "filetypes", required = false) String filetypes) { ++ @RequestParam(value = "filetypes", required = false) String filetypes ++ ) { return ok(FileManager.getAllFiles(filetypes)); } @@@ -109,24 -108,22 +111,33 @@@ description = "Determines if the provided file name is the original file name " + "as per upload." ) - @QueryParam("isOriginalFilename") @DefaultValue("false") boolean isOriginalFilename + @RequestParam(value = "isOriginalFilename", defaultValue = "false") boolean isOriginalFilename ) { - if (isOriginalFilename) { - try { - return ok(FileManager.getFileByOriginalName(filename)); - } catch (IOException e) { - return notFound(filename); + try { + if (isOriginalFilename) { + return ok(getFileContents(FileManager.getFileByOriginalName(filename))); + } else { + return ok(getFileContents(FileManager.getFile(filename))); } + } catch (IOException e) { + throw new SpMessageException( + org.springframework.http.HttpStatus.NOT_FOUND, - Notifications.error("File not found")); ++ Notifications.error("File not found") ++ ); } - return ok(FileManager.getFile(filename)); } - @GET - @Path("/allFilenames") - @Produces(MediaType.APPLICATION_JSON) + private byte[] getFileContents(File file) throws IOException { + return Files.readAllBytes(file.toPath()); + } ++ ++ @GetMapping(path = "/allFilenames", produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize(AuthConstants.HAS_READ_FILE_PRIVILEGE) - public Response getAllOriginalFilenames() { - return ok(FileManager.getAllFiles().stream().map(fileMetadata -> fileMetadata.getOriginalFilename().toLowerCase()) - .toList(); ++ public ResponseEntity<List<String>> getAllOriginalFilenames() { ++ return ok(FileManager.getAllFiles() ++ .stream() ++ .map(fileMetadata -> fileMetadata.getOriginalFilename() ++ .toLowerCase()) ++ .toList()); + } }
