This is an automated email from the ASF dual-hosted git repository. riemer pushed a commit to branch STREAMPIPES-319 in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 523925e1f18c00a95273eebaa72fce958b57a460 Merge: 8e6ddcf 2115359 Author: Dominik Riemer <[email protected]> AuthorDate: Thu May 27 00:05:44 2021 +0200 Merge branch 'dev' into STREAMPIPES-319 .../backend/StreamPipesBackendApplication.java | 117 ++++++++++--- .../backend/StreamPipesResourceConfig.java | 3 + .../streampipes/client/api/DataProcessorApi.java | 41 +++++ .../apache/streampipes/client/api/DataSinkApi.java | 28 +++ .../streampipes/client/model/InputStreamIndex.java | 23 ++- .../master/management/AdapterMasterManagement.java | 65 +++++-- .../management/WorkerAdministrationManagement.java | 4 +- .../master/management/WorkerRestClient.java | 123 ++++--------- .../master/rest/AbstractAdapterResource.java | 20 +-- .../container/master/rest/AdapterResource.java | 93 ++++++---- .../master/rest/AdapterTemplateResource.java | 19 +- .../container/master/rest/DescriptionResource.java | 59 +++---- .../container/master/rest/GuessResource.java | 32 ++-- .../master/rest/RuntimeResolvableResource.java | 8 +- .../container/master/rest/SourcesResource.java | 33 ++-- .../container/master/rest/UnitResource.java | 17 +- .../container/master/rest/WelcomePageMaster.java | 46 ++--- .../master/util/AdapterEncryptionService.java | 33 +--- .../management}/AdapterMasterManagementTest.java | 31 ++-- .../AdapterTemplateMasterManagementTest.java | 14 +- .../management}/DescriptionManagementTest.java | 19 +- .../master/management/SourcesManagementTest.java | 0 .../management/UnitMasterManagementTest.java | 0 .../master/management/WorkerRestClientTest.java | 35 +--- .../api/InvocablePipelineElementResource.java | 13 ++ .../container/init/RunningInstances.java | 14 ++ .../streampipes/model/canvas/CanvasPosition.java | 25 ++- .../model/canvas/PipelineCanvasComment.java | 29 ++-- .../PipelineCanvasMetadata.java} | 64 ++++--- .../model/canvas/PipelineElementMetadata.java | 29 ++-- .../model/connect/adapter/AdapterDescription.java | 13 ++ .../connect/adapter/AdapterStreamDescription.java | 11 ++ .../model/dashboard/DashboardEntity.java | 1 - .../streampipes/model/dashboard/DashboardItem.java | 11 +- .../model/dashboard/DashboardWidgetModel.java | 47 +---- .../model/dashboard/VisualizablePipeline.java | 20 +-- .../streampipes/model/pipeline/Pipeline.java | 22 +++ .../model/pipeline/PipelineHealthStatus.java | 21 +-- .../model/preview/PipelinePreviewModel.java | 36 ++-- .../manager/execution/http/GraphSubmitter.java | 5 +- .../manager/execution/http/PipelineExecutor.java | 59 +++---- .../execution/http/PipelineStorageService.java | 88 ++++------ .../health/PipelineElementEndpointHealthCheck.java | 49 ++++++ .../manager/health/PipelineHealthCheck.java | 193 +++++++++++++++++++++ .../matching/PipelineVerificationHandler.java | 21 ++- .../monitoring/pipeline/TopicInfoCollector.java | 5 +- .../streampipes/manager/operations/Operations.java | 20 ++- .../manager/preview/ActivePipelinePreviews.java | 63 +++++++ .../manager/preview/PipelinePreview.java | 125 +++++++++++++ .../streampipes/manager/secret/ISecretHandler.java | 17 +- .../manager/secret/SecretDecrypter.java | 30 ++-- .../manager/secret/SecretEncrypter.java} | 30 ++-- .../streampipes/manager/secret/SecretProvider.java | 21 ++- .../streampipes/manager/secret/SecretService.java | 52 ++++++ .../streampipes/manager/secret/SecretVisitor.java | 116 +++++++++++++ .../manager/setup/CouchDbInstallationStep.java | 1 - .../rest/impl/AbstractRestResource.java | 10 -- .../rest/impl/PipelineCanvasMetadataCache.java | 54 ++++++ .../rest/impl/PipelineCanvasMetadataResource.java | 106 +++++++++++ .../rest/impl/PipelineElementPreview.java | 64 +++++++ .../streampipes/rest/impl/PipelineResource.java | 20 ++- .../rest/impl/dashboard/VisualizablePipeline.java | 94 ++++++---- .../impl/nouser/PipelineElementImportNoUser.java | 12 +- .../rest/impl/nouser/PipelineNoUserResource.java | 2 +- .../rest/management/PipelineManagement.java | 5 +- .../serializers/json/AdapterSerializer.java | 14 +- .../streampipes/storage/api/IAdapterStorage.java | 2 +- .../streampipes/storage/api/INoSqlStorage.java | 8 +- ...ge.java => IPipelineCanvasMetadataStorage.java} | 11 +- .../storage/couchdb/CouchDbStorageManager.java | 20 ++- .../storage/couchdb/impl/AdapterStorageImpl.java | 7 +- .../impl/PipelineCanvasMetadataStorageImpl.java | 69 ++++++++ .../streampipes/storage/couchdb/utils/Utils.java | 113 +++++------- streampipes-wrapper-siddhi/pom.xml | 17 +- ui/src/app/add/add.component.ts | 22 ++- .../endpoint-item/endpoint-item.component.html | 2 +- .../adapter-description.component.html | 15 +- .../adapter-description.component.ts | 10 -- .../data-marketplace.component.html | 62 ++++--- .../data-marketplace/data-marketplace.component.ts | 16 +- .../existing-adapters.component.html | 126 ++++++++++++++ .../existing-adapters.component.scss} | 15 +- .../existing-adapters.component.ts | 96 ++++++++++ ui/src/app/connect/connect.module.ts | 2 + .../delete-adapter-dialog.component.html | 52 ++++++ .../delete-adapter-dialog.component.ts | 64 +++++++ .../delete-pipeline-dialog.component.scss} | 18 +- .../connect/services/data-marketplace.service.ts | 37 ++-- ui/src/app/core-model/gen/streampipes-model.ts | 125 ++++++++++--- .../components/grid/dashboard-grid.component.ts | 2 - .../widget/dashboard-widget.component.css | 3 +- .../widget/dashboard-widget.component.html | 51 +++--- .../widget/dashboard-widget.component.ts | 147 ++++++++++------ ui/src/app/dashboard/dashboard.component.ts | 9 +- ui/src/app/dashboard/dashboard.module.ts | 4 + .../add-visualization-dialog.component.html | 5 +- .../add-visualization-dialog.component.ts | 66 ++++--- ui/src/app/dashboard/services/dashboard.service.ts | 19 +- .../{resize.service.ts => edit-mode.service.ts} | 14 +- .../services/refresh-dashboard.service.ts | 6 +- ...esize.service.ts => reload-pipeline.service.ts} | 14 +- ui/src/app/dashboard/services/resize.service.ts | 6 +- .../data-explorer-dashboard-widget.component.html | 8 +- .../pipeline-assembly.component.html | 19 +- .../pipeline-assembly.component.scss | 10 ++ .../pipeline-assembly.component.ts | 77 +++++--- .../pipeline-element-options.component.html | 7 +- .../pipeline-element-preview.component.html | 32 ++++ .../pipeline-element-preview.component.scss} | 27 +-- .../pipeline-element-preview.component.ts | 68 ++++++++ .../components/pipeline/pipeline.component.html | 11 +- .../components/pipeline/pipeline.component.ts | 125 ++++++++----- .../save-pipeline/save-pipeline.component.html | 4 +- .../save-pipeline/save-pipeline.component.ts | 43 +++-- ui/src/app/editor/editor.component.html | 2 +- ui/src/app/editor/editor.component.scss | 4 +- ui/src/app/editor/editor.module.ts | 2 + ui/src/app/editor/services/editor.service.ts | 41 ++++- ui/src/app/editor/services/jsplumb.service.ts | 24 ++- .../services/pipeline-positioning.service.ts | 76 ++++++-- .../actions/pipeline-actions.component.html | 3 +- .../pipeline-overview.component.html | 19 ++ .../pipeline-overview.component.scss | 43 ++++- .../pipeline-overview.component.ts | 7 +- .../pipeline-notifications.component.html | 35 ++++ .../pipeline-notifications.component.scss} | 16 +- .../pipeline-notifications.component.ts | 53 ++++++ .../pipeline-status-dialog.component.html | 65 +++++-- .../pipeline-status-dialog.component.scss | 34 +++- .../pipeline-status-dialog.component.ts | 68 +++++++- .../model/pipeline-model.ts} | 19 +- ui/src/app/pipelines/pipelines.component.html | 6 + ui/src/app/pipelines/pipelines.module.ts | 4 +- .../services/pipeline-operations.service.ts | 77 +++++--- .../apis/pipeline-canvas-metadata.service.ts | 65 +++++++ .../app/platform-services/apis/pipeline.service.ts | 13 +- ui/src/app/platform-services/platform.module.ts | 2 + ui/src/scss/sp/pipeline-element-options.scss | 14 +- 138 files changed, 3511 insertions(+), 1362 deletions(-) diff --cc streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java index d99c090,faf6ffd..7759c4d --- a/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java +++ b/streampipes-backend/src/main/java/org/apache/streampipes/backend/StreamPipesBackendApplication.java @@@ -20,14 -20,13 +20,16 @@@ package org.apache.streampipes.backend import org.apache.shiro.web.env.EnvironmentLoaderListener; import org.apache.shiro.web.servlet.OncePerRequestFilter; import org.apache.shiro.web.servlet.ShiroFilter; +import org.apache.streampipes.commons.networking.Networking; + import org.apache.streampipes.manager.health.PipelineHealthCheck; import org.apache.streampipes.manager.operations.Operations; import org.apache.streampipes.model.pipeline.Pipeline; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; import org.apache.streampipes.rest.notifications.NotificationListener; + import org.apache.streampipes.storage.api.IPipelineStorage; import org.apache.streampipes.storage.management.StorageDispatcher; +import org.apache.streampipes.svcdiscovery.SpServiceDiscovery; +import org.apache.streampipes.svcdiscovery.SpServiceTags; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; @@@ -41,9 -40,9 +43,11 @@@ import org.springframework.context.anno import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.servlet.ServletContextListener; + import java.util.HashMap; +import java.net.UnknownHostException; +import java.util.Arrays; import java.util.List; + import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@@ -55,23 -55,20 +60,34 @@@ public class StreamPipesBackendApplicat private static final Logger LOG = LoggerFactory.getLogger(StreamPipesBackendApplication.class.getCanonicalName()); + private static final int MAX_PIPELINE_START_RETRIES = 3; + private static final int WAIT_TIME_AFTER_FAILURE_IN_SECONDS = 10; + + private static final int HEALTH_CHECK_INTERVAL = 60; + private static final TimeUnit HEALTH_CHECK_UNIT = TimeUnit.SECONDS; + + private ScheduledExecutorService executorService; + private ScheduledExecutorService healthCheckExecutorService; + + private Map<String, Integer> failedPipelines = new HashMap<>(); + public static void main(String[] args) { System.setProperty("org.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH", "true"); - SpringApplication.run(StreamPipesBackendApplication.class, args); + try { + String host = Networking.getHostname(); + Integer port = Networking.getPort(8030); + SpServiceDiscovery + .getServiceDiscovery() + .registerService("core", + "core", + host, + port, + Arrays.asList(SpServiceTags.CORE, SpServiceTags.CONNECT_MASTER)); + + SpringApplication.run(StreamPipesBackendApplication.class, args); + } catch (UnknownHostException e) { + LOG.error("Could not auto-resolve host address - please manually provide the hostname using the SP_HOST environment variable"); + } } @PostConstruct diff --cc streampipes-container/src/main/java/org/apache/streampipes/container/api/InvocablePipelineElementResource.java index 1adce39,bbcdec3..f391685 --- a/streampipes-container/src/main/java/org/apache/streampipes/container/api/InvocablePipelineElementResource.java +++ b/streampipes-container/src/main/java/org/apache/streampipes/container/api/InvocablePipelineElementResource.java @@@ -71,7 -70,13 +71,13 @@@ public abstract class InvocablePipeline String runningInstanceId = getInstanceId(graph.getElementId(), elementId); if (!RunningInstances.INSTANCE.exists(runningInstanceId)) { RunningInstances.INSTANCE.add(runningInstanceId, graph, declarer.getClass().newInstance()); - Response resp = RunningInstances.INSTANCE.getInvocation(runningInstanceId).invokeRuntime(graph); + Response resp = RunningInstances.INSTANCE.getInvocation(runningInstanceId).invokeRuntime(graph, getServiceId()); + if (!resp.isSuccess()) { + LOG.error("Could not invoke pipeline element {} due to the following error: {}", + graph.getName(), + resp.getOptionalMessage()); + RunningInstances.INSTANCE.remove(runningInstanceId); + } return ok(resp); } else { LOG.info("Pipeline element {} with id {} seems to be already running, skipping invocation request.", graph.getName(), runningInstanceId); diff --cc streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/GraphSubmitter.java index cf56826,3eddef1..5b3f34c --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/GraphSubmitter.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/GraphSubmitter.java @@@ -23,9 -24,8 +23,10 @@@ import org.apache.streampipes.model.SpD import org.apache.streampipes.model.base.InvocableStreamPipesEntity; import org.apache.streampipes.model.pipeline.PipelineElementStatus; import org.apache.streampipes.model.pipeline.PipelineOperationStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.ArrayList; import java.util.List; import java.util.Optional; diff --cc streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/PipelineExecutor.java index c01371d,5775753..7ac5c72 --- a/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/PipelineExecutor.java +++ b/streampipes-pipeline-management/src/main/java/org/apache/streampipes/manager/execution/http/PipelineExecutor.java @@@ -18,8 -18,8 +18,9 @@@ package org.apache.streampipes.manager.execution.http; +import org.apache.streampipes.commons.constants.PipelineElementUrl; import org.apache.streampipes.manager.execution.status.PipelineStatusManager; + import org.apache.streampipes.manager.secret.SecretProvider; import org.apache.streampipes.manager.util.TemporaryGraphStorage; import org.apache.streampipes.model.SpDataSet; import org.apache.streampipes.model.base.InvocableStreamPipesEntity; @@@ -76,17 -77,16 +78,19 @@@ public class PipelineExecutor graphs.addAll(sepas); graphs.addAll(secs); + decryptSecrets(graphs); + - //graphs.forEach(g -> g.setStreamRequirements(Collections.emptyList())); + graphs.forEach(g -> g.setSelectedEndpointUrl(new PipelineElementEndpointGenerator(g.getElementId(), + g.getAppId(), + getPipelineElementType(g)) + .getEndpointResourceUrl())); - List<InvocableStreamPipesEntity> decryptedGraphs = decryptSecrets(graphs); - PipelineOperationStatus status = new GraphSubmitter(pipeline.getPipelineId(), - pipeline.getName(), decryptedGraphs, dataSets) + pipeline.getName(), graphs, dataSets) .invokeGraphs(); + encryptSecrets(graphs); + if (status.isSuccess()) { storeInvocationGraphs(pipeline.getPipelineId(), graphs, dataSets);
