This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch
3525-extend-capabilities-of-asset-export-feature
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/3525-extend-capabilities-of-asset-export-feature by this push:
new 5617775376 feat(#3525): Extend capabilities of asset export
5617775376 is described below
commit 5617775376793c151cea0f8e4bf2f1ed7ad47dc8
Author: Dominik Riemer <[email protected]>
AuthorDate: Tue Mar 11 08:32:18 2025 +0100
feat(#3525): Extend capabilities of asset export
---
streampipes-data-export/pom.xml | 5 ++
.../export/dataimport/ImportGenerator.java | 11 +++
.../export/dataimport/PerformImportGenerator.java | 53 ++++++++----
.../export/dataimport/PreviewImportGenerator.java | 9 +-
.../export/generator/ExportPackageGenerator.java | 5 ++
.../export/resolver/AbstractResolver.java | 21 ++---
.../export/resolver/AdapterResolver.java | 44 ++++++++--
.../streampipes/export/resolver/ChartResolver.java | 12 ++-
.../export/resolver/DashboardResolver.java | 12 ++-
.../export/resolver/DataSourceResolver.java | 18 ++--
.../export/resolver/DocumentResolver.java | 29 ++++---
.../streampipes/export/resolver/FileResolver.java | 12 ++-
.../resolver/GenericStorageDocumentResolver.java | 97 ++++++++++++++++++++++
.../export/resolver/MeasurementResolver.java | 12 ++-
.../export/resolver/PipelineResolver.java | 33 +++++---
.../export/utils/ImportAdapterMigrationUtils.java | 55 ------------
.../model/export/AssetExportConfiguration.java | 23 +++++
.../export/StreamPipesApplicationPackage.java | 14 ++++
.../streampipes/storage/api/CRUDStorage.java | 4 +-
.../src/lib/model/gen/streampipes-model.ts | 13 ++-
ui/src/app/configuration/configuration.module.ts | 4 +
.../data-export-dialog.component.html | 4 +
.../generic-storage-item.component.html | 39 +++++++++
.../generic-storage-item.component.ts | 31 +++----
.../generic-storage-items.component.html | 54 ++++++++++++
.../generic-storage-items.component.ts | 36 ++++++++
.../data-import-dialog.component.html | 46 ++++++++--
.../data-import-dialog.component.scss | 5 ++
.../import-dialog/data-import-dialog.component.ts | 30 ++++++-
29 files changed, 569 insertions(+), 162 deletions(-)
diff --git a/streampipes-data-export/pom.xml b/streampipes-data-export/pom.xml
index 46e1b7837b..5bbf50542a 100644
--- a/streampipes-data-export/pom.xml
+++ b/streampipes-data-export/pom.xml
@@ -37,6 +37,11 @@
<artifactId>streampipes-pipeline-management</artifactId>
<version>0.98.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-connect-management</artifactId>
+ <version>0.98.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>org.apache.streampipes</groupId>
<artifactId>streampipes-resource-management</artifactId>
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
index 11e3340148..8ca45ff1f0 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/ImportGenerator.java
@@ -115,6 +115,14 @@ public abstract class ImportGenerator<T> {
}
}
+ for (String documentId : manifest.getGenericStorageDocuments()) {
+ try {
+ handleGenericStorageDocument(asString(previewFiles.get(documentId)),
documentId);
+ } catch (DocumentConflictException e) {
+ LOG.warn("Skipping import of generic storage doc {} (already present
with the same id)", documentId);
+ }
+ }
+
afterResourcesCreated();
return getReturnObject();
@@ -147,6 +155,9 @@ public abstract class ImportGenerator<T> {
protected abstract void handleFile(String document, String fileMetadataId,
Map<String, byte[]> zipContent)
throws IOException;
+ protected abstract void handleGenericStorageDocument(String document, String
dataLakeMeasureId)
+ throws JsonProcessingException;
+
protected abstract T getReturnObject();
protected abstract void afterResourcesCreated();
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
index 580341c996..674d9b287e 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
@@ -19,14 +19,15 @@
package org.apache.streampipes.export.dataimport;
import org.apache.streampipes.export.model.PermissionInfo;
+import org.apache.streampipes.export.resolver.AbstractResolver;
import org.apache.streampipes.export.resolver.AdapterResolver;
import org.apache.streampipes.export.resolver.ChartResolver;
import org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
+import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
-import org.apache.streampipes.export.utils.ImportAdapterMigrationUtils;
import org.apache.streampipes.manager.file.FileHandler;
import org.apache.streampipes.model.SpDataStream;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
@@ -64,14 +65,22 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleAsset(Map<String, byte[]> previewFiles, String assetId)
throws IOException {
- storage.getGenericStorage().create(asString(previewFiles.get(assetId)));
+ var document = asString(previewFiles.get(assetId));
+ try {
+ var existing = storage.getGenericStorage().findOne(assetId);
+ if (config.isOverwriteExistingDocuments()) {
+ storage.getGenericStorage().delete(assetId,
existing.get("_rev").toString());
+ }
+ } catch (IOException e) {
+ // Document not found, do nothing
+ }
+ storage.getGenericStorage().create(document);
}
@Override
protected void handleAdapter(String document, String adapterId) throws
JsonProcessingException {
if (shouldStore(adapterId, config.getAdapters())) {
- var convertedDoc =
ImportAdapterMigrationUtils.checkAndPerformMigration(document);
- new AdapterResolver().writeDocument(convertedDoc,
config.isOverrideBrokerSettings());
+ writeDocument(document, new AdapterResolver());
permissionsToStore.add(new PermissionInfo(adapterId,
AdapterDescription.class));
}
}
@@ -79,7 +88,7 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleChart(String document, String chartId) throws
JsonProcessingException {
if (shouldStore(chartId, config.getDataViews())) {
- new ChartResolver().writeDocument(document);
+ writeDocument(document, new ChartResolver());
permissionsToStore.add(new PermissionInfo(chartId,
DataExplorerWidgetModel.class));
}
}
@@ -87,7 +96,7 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleDashboard(String document, String dashboardId) throws
JsonProcessingException {
if (shouldStore(dashboardId, config.getDashboards())) {
- new DashboardResolver().writeDocument(document);
+ writeDocument(document, new DashboardResolver());
permissionsToStore.add(new PermissionInfo(dashboardId,
DashboardModel.class));
}
}
@@ -95,7 +104,7 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleDataSource(String document, String dataSourceId) throws
JsonProcessingException {
if (shouldStore(dataSourceId, config.getDataSources())) {
- new DataSourceResolver().writeDocument(document,
config.isOverrideBrokerSettings());
+ writeDocument(document, new DataSourceResolver());
permissionsToStore.add(new PermissionInfo(dataSourceId,
SpDataStream.class));
}
}
@@ -103,7 +112,7 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handlePipeline(String document, String pipelineId) throws
JsonProcessingException {
if (shouldStore(pipelineId, config.getPipelines())) {
- new PipelineResolver().writeDocument(document,
config.isOverrideBrokerSettings());
+ writeDocument(document, new PipelineResolver());
permissionsToStore.add(new PermissionInfo(pipelineId, Pipeline.class));
}
}
@@ -111,7 +120,7 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleDataLakeMeasure(String document, String
dataLakeMeasureId) throws JsonProcessingException {
if (shouldStore(dataLakeMeasureId, config.getDataLakeMeasures())) {
- new MeasurementResolver().writeDocument(document);
+ writeDocument(document, new MeasurementResolver());
permissionsToStore.add(new PermissionInfo(dataLakeMeasureId,
DataLakeMeasure.class));
}
}
@@ -122,12 +131,27 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
Map<String, byte[]> zipContent) throws IOException
{
var resolver = new FileResolver();
var fileMetadata = resolver.readDocument(document);
- resolver.writeDocument(document);
+ writeDocument(document, resolver);
byte[] file = zipContent.get(
fileMetadata.getFilename().substring(0,
fileMetadata.getFilename().lastIndexOf(".")));
new FileHandler().storeFile(fileMetadata.getFilename(), new
ByteArrayInputStream(file));
}
+ @Override
+ protected void handleGenericStorageDocument(String document, String
documentId) throws JsonProcessingException {
+ if (shouldStore(documentId, config.getGenericStorageDocuments())) {
+ writeDocument(document, new GenericStorageDocumentResolver());
+ }
+ }
+
+ private void writeDocument(String document,
+ AbstractResolver<?> resolver) throws
JsonProcessingException {
+ if (config.isOverwriteExistingDocuments()) {
+ resolver.deleteDocument(document);
+ }
+ resolver.writeDocument(document, config);
+ }
+
@Override
protected Void getReturnObject() {
return null;
@@ -144,12 +168,11 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
true));
}
- private boolean shouldStore(String adapterId,
- Set<ExportItem> adapters) {
- return adapters
+ private boolean shouldStore(String documentId,
+ Set<ExportItem> exportItems) {
+ return exportItems
.stream()
- .filter(item -> item.getResourceId().equals(adapterId))
+ .filter(item -> item.getResourceId().equals(documentId))
.allMatch(ExportItem::isSelected);
}
-
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
index 6d7383a9d5..40297b265c 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
@@ -26,7 +26,6 @@ import
org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
-import org.apache.streampipes.export.utils.ImportAdapterMigrationUtils;
import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
@@ -69,8 +68,7 @@ public class PreviewImportGenerator extends
ImportGenerator<AssetExportConfigura
protected void handleAdapter(String document,
String adapterId) throws
JsonProcessingException {
try {
- var convertedDoc =
ImportAdapterMigrationUtils.checkAndPerformMigration(document);
- addExportItem(adapterId, new
AdapterResolver().readDocument(convertedDoc).getName(),
importConfig::addAdapter);
+ addExportItem(adapterId, new
AdapterResolver().readDocument(document).getName(), importConfig::addAdapter);
} catch (IllegalArgumentException e) {
LOG.warn("Skipping import of data set adapter {}", adapterId);
}
@@ -114,6 +112,11 @@ public class PreviewImportGenerator extends
ImportGenerator<AssetExportConfigura
importConfig::addFile);
}
+ @Override
+ protected void handleGenericStorageDocument(String document, String
genericDocId) throws JsonProcessingException {
+ addExportItem(genericDocId, genericDocId,
importConfig::addGenericStorageDocument);
+ }
+
@Override
protected AssetExportConfiguration getReturnObject() {
return this.importConfig;
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
index 752ce4209b..a5b6b1e926 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -25,6 +25,7 @@ import org.apache.streampipes.export.resolver.ChartResolver;
import org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
+import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
import org.apache.streampipes.export.utils.SerializationUtils;
@@ -108,6 +109,10 @@ public class ExportPackageGenerator {
manifest::addDataView);
});
+ config.getGenericStorageDocuments().forEach(item -> {
+ addDoc(builder, item, new GenericStorageDocumentResolver(),
manifest::addGenericStorageDocument);
+ });
+
config.getFiles().forEach(item -> {
var fileResolver = new FileResolver();
String filename =
fileResolver.findDocument(item.getResourceId()).getFilename();
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
index d86acdba29..1b59196125 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
@@ -29,13 +29,12 @@ import
org.apache.streampipes.storage.management.StorageDispatcher;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import org.lightcouch.DocumentConflictException;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-public abstract class AbstractResolver<T> {
+public abstract class AbstractResolver<T> implements DocumentResolver<T> {
protected ObjectMapper spMapper;
protected ObjectMapper defaultMapper;
@@ -61,7 +60,7 @@ public abstract class AbstractResolver<T> {
public String getSerializedDocument(String resourceId) throws
JsonProcessingException, ElementNotFoundException {
var document = findDocument(resourceId);
if (document != null) {
- return
SerializationUtils.getSpObjectMapper().writeValueAsString(modifyDocumentForExport(document));
+ return
getObjectMapper().writeValueAsString(modifyDocumentForExport(document));
} else {
throw new ElementNotFoundException("Could not find element with resource
id " + resourceId);
}
@@ -71,20 +70,12 @@ public abstract class AbstractResolver<T> {
return StorageDispatcher.INSTANCE.getNoSqlStore();
}
- public abstract T findDocument(String resourceId);
-
- public abstract T modifyDocumentForExport(T doc);
-
- public abstract T readDocument(String serializedDoc) throws
JsonProcessingException;
-
- public abstract ExportItem convert(T document);
-
- public abstract void writeDocument(String document) throws
JsonProcessingException, DocumentConflictException;
-
- protected abstract T deserializeDocument(String document) throws
JsonProcessingException;
-
protected void overrideProtocol(EventGrounding grounding) {
var newProtocol = new
EventGroundingProcessor().applyOverride(grounding.getTransportProtocol());
grounding.setTransportProtocol(newProtocol);
}
+
+ protected ObjectMapper getObjectMapper() {
+ return spMapper;
+ }
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
index d22ae5ba72..0fdb6329d3 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AdapterResolver.java
@@ -19,14 +19,24 @@
package org.apache.streampipes.export.resolver;
+import org.apache.streampipes.commons.exceptions.connect.AdapterException;
+import org.apache.streampipes.commons.prometheus.adapter.AdapterMetricsManager;
+import
org.apache.streampipes.connect.management.management.AdapterMasterManagement;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.connect.adapter.AdapterDescription;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.SpResourceManager;
+import org.apache.streampipes.resource.management.secret.SecretProvider;
import com.fasterxml.jackson.core.JsonProcessingException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class AdapterResolver extends AbstractResolver<AdapterDescription> {
+ private static final Logger LOG =
LoggerFactory.getLogger(AdapterResolver.class);
+
@Override
public AdapterDescription findDocument(String resourceId) {
return
getNoSqlStore().getAdapterInstanceStorage().getElementById(resourceId);
@@ -37,6 +47,7 @@ public class AdapterResolver extends
AbstractResolver<AdapterDescription> {
adapterDescription.setRev(null);
adapterDescription.setSelectedEndpointUrl(null);
adapterDescription.setRunning(false);
+ SecretProvider.getDecryptionService().apply(adapterDescription);
return adapterDescription;
}
@@ -52,22 +63,41 @@ public class AdapterResolver extends
AbstractResolver<AdapterDescription> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
-
getNoSqlStore().getAdapterInstanceStorage().persist(deserializeDocument(document));
- }
-
public void writeDocument(String document,
- boolean overrideDocument) throws
JsonProcessingException {
+ AssetExportConfiguration config) throws
JsonProcessingException {
var adapterDescription = deserializeDocument(document);
- if (overrideDocument) {
+ if (config.isOverrideBrokerSettings()) {
overrideProtocol(adapterDescription.getEventGrounding());
}
+ SecretProvider.getEncryptionService().apply(adapterDescription);
getNoSqlStore().getAdapterInstanceStorage().persist(adapterDescription);
}
@Override
- protected AdapterDescription deserializeDocument(String document) throws
JsonProcessingException {
+ public AdapterDescription deserializeDocument(String document) throws
JsonProcessingException {
return this.spMapper.readValue(document, AdapterDescription.class);
}
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var adapter = deserializeDocument(document);
+ var resourceId = adapter.getElementId();
+ var existingAdapter =
getNoSqlStore().getAdapterInstanceStorage().getElementById(resourceId);
+ if (existingAdapter != null) {
+ if (existingAdapter.isRunning()) {
+ try {
+ new AdapterMasterManagement(
+ getNoSqlStore().getAdapterInstanceStorage(),
+ new SpResourceManager().manageAdapters(),
+ new SpResourceManager().manageDataStreams(),
+ AdapterMetricsManager.INSTANCE.getAdapterMetrics()
+ ).stopStreamAdapter(resourceId);
+ } catch (AdapterException e) {
+ LOG.warn("Error when stopping adapter with id {} and name {}",
resourceId, existingAdapter.getName());
+ }
+ }
+
getNoSqlStore().getAdapterInstanceStorage().deleteElementById(resourceId);
+ }
+ }
+
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
index 4b65871510..212a3567cc 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/ChartResolver.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.datalake.DataExplorerWidgetModel;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -51,13 +52,20 @@ public class ChartResolver extends
AbstractResolver<DataExplorerWidgetModel> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
+ public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException {
getNoSqlStore().getDataExplorerWidgetStorage().persist(deserializeDocument(document));
}
@Override
- protected DataExplorerWidgetModel deserializeDocument(String document)
throws JsonProcessingException {
+ public DataExplorerWidgetModel deserializeDocument(String document) throws
JsonProcessingException {
return this.defaultMapper.readValue(document,
DataExplorerWidgetModel.class);
}
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var chart = readDocument(document);
+ var resourceId = chart.getElementId();
+
getNoSqlStore().getDataExplorerWidgetStorage().deleteElementById(resourceId);
+ }
+
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
index 6c7439c1b3..d7cd1c128c 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DashboardResolver.java
@@ -22,6 +22,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.dashboard.DashboardItem;
import org.apache.streampipes.model.dashboard.DashboardModel;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -59,15 +60,22 @@ public class DashboardResolver extends
AbstractResolver<DashboardModel> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
+ public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException {
getNoSqlStore().getDataExplorerDashboardStorage().persist(deserializeDocument(document));
}
@Override
- protected DashboardModel deserializeDocument(String document) throws
JsonProcessingException {
+ public DashboardModel deserializeDocument(String document) throws
JsonProcessingException {
return this.spMapper.readValue(document, DashboardModel.class);
}
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var dashboard = readDocument(document);
+ var resourceId = dashboard.getElementId();
+
getNoSqlStore().getDataExplorerDashboardStorage().deleteElementById(resourceId);
+ }
+
public List<String> getCharts(String resourceId) {
var document = findDocument(resourceId);
return
document.getWidgets().stream().map(DashboardItem::getId).collect(Collectors.toList());
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
index a20aa0b2a9..9072dd9682 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DataSourceResolver.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.SpDataStream;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -48,14 +49,10 @@ public class DataSourceResolver extends
AbstractResolver<SpDataStream> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
-
getNoSqlStore().getDataStreamStorage().persist(deserializeDocument(document));
- }
-
public void writeDocument(String document,
- boolean overrideDocument) throws
JsonProcessingException {
+ AssetExportConfiguration config) throws
JsonProcessingException {
var dataStream = deserializeDocument(document);
- if (overrideDocument) {
+ if (config.isOverrideBrokerSettings()) {
if (dataStream.getEventGrounding() != null) {
overrideProtocol(dataStream.getEventGrounding());
}
@@ -64,7 +61,14 @@ public class DataSourceResolver extends
AbstractResolver<SpDataStream> {
}
@Override
- protected SpDataStream deserializeDocument(String document) throws
JsonProcessingException {
+ public SpDataStream deserializeDocument(String document) throws
JsonProcessingException {
return this.spMapper.readValue(document, SpDataStream.class);
}
+
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var dataSource = readDocument(document);
+ var resourceId = dataSource.getElementId();
+ getNoSqlStore().getDataStreamStorage().deleteElementById(resourceId);
+ }
}
diff --git
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
similarity index 50%
copy from
streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
copy to
streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
index 32e57df0af..2e4b4e35dc 100644
---
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/DocumentResolver.java
@@ -15,26 +15,29 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.storage.api;
-import org.apache.streampipes.model.Tuple2;
+package org.apache.streampipes.export.resolver;
-import java.util.List;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.model.export.ExportItem;
-public interface CRUDStorage<T> {
+import com.fasterxml.jackson.core.JsonProcessingException;
+import org.lightcouch.DocumentConflictException;
- List<T> findAll();
+public interface DocumentResolver<T> {
- Tuple2<Boolean, String> persist(T element);
+ T findDocument(String resourceId);
- T getElementById(String id);
+ T modifyDocumentForExport(T document);
- T updateElement(T element);
+ T readDocument(String serializedDocument) throws JsonProcessingException;
- void deleteElement(T element);
+ ExportItem convert(T document);
- default void deleteElementById(String id) {
- var element = getElementById(id);
- deleteElement(element);
- }
+ void writeDocument(String document, AssetExportConfiguration config)
+ throws JsonProcessingException, DocumentConflictException;
+
+ T deserializeDocument(String document) throws JsonProcessingException;
+
+ void deleteDocument(String document) throws JsonProcessingException;
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
index ab0179d8f1..f744c73465 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/FileResolver.java
@@ -19,6 +19,7 @@
package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.file.FileMetadata;
@@ -48,12 +49,19 @@ public class FileResolver extends
AbstractResolver<FileMetadata> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
+ public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException {
getNoSqlStore().getFileMetadataStorage().persist(deserializeDocument(document));
}
@Override
- protected FileMetadata deserializeDocument(String document) throws
JsonProcessingException {
+ public FileMetadata deserializeDocument(String document) throws
JsonProcessingException {
return SerializationUtils.getSpObjectMapper().readValue(document,
FileMetadata.class);
}
+
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var fileMetadata = readDocument(document);
+ var resourceId = fileMetadata.getElementId();
+ getNoSqlStore().getFileMetadataStorage().deleteElementById(resourceId);
+ }
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
new file mode 100644
index 0000000000..da4a50162b
--- /dev/null
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.resolver;
+
+import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.model.export.ExportItem;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.lightcouch.DocumentConflictException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class GenericStorageDocumentResolver extends
AbstractResolver<Map<String, Object>> {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(GenericStorageDocumentResolver.class);
+
+ @Override
+ public Map<String, Object> findDocument(String resourceId) {
+ try {
+ return getNoSqlStore().getGenericStorage().findOne(resourceId);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ @Override
+ public Map<String, Object> modifyDocumentForExport(Map<String, Object>
document) {
+ document.remove("_rev");
+ return document;
+ }
+
+ @Override
+ public Map<String, Object> readDocument(String serializedDocument) throws
JsonProcessingException {
+ return
SerializationUtils.getDefaultObjectMapper().readValue(serializedDocument, new
TypeReference<>() {});
+ }
+
+ @Override
+ public ExportItem convert(Map<String, Object> document) {
+ var documentId = document.get("_id").toString();
+ return new ExportItem(documentId, documentId, true);
+ }
+
+ @Override
+ public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException, DocumentConflictException {
+ try {
+ getNoSqlStore().getGenericStorage().create(document);
+ } catch (IOException e) {
+ LOG.warn("Could not write document");
+ }
+ }
+
+ @Override
+ public Map<String, Object> deserializeDocument(String document) throws
JsonProcessingException {
+ return SerializationUtils.getDefaultObjectMapper().readValue(document, new
TypeReference<>() {});
+ }
+
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ try {
+ var asset = readDocument(document);
+ var resourceId = asset.get("_id").toString();
+ var storedAsset =
getNoSqlStore().getGenericStorage().findOne(resourceId);
+ getNoSqlStore().getGenericStorage().delete(resourceId,
storedAsset.get("_rev").toString());
+ } catch (IOException e) {
+ // Do nothing
+ }
+ }
+
+ @Override
+ protected ObjectMapper getObjectMapper() {
+ return defaultMapper;
+ }
+
+
+}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
index e94012c91d..6e1373c073 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/MeasurementResolver.java
@@ -20,6 +20,7 @@ package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -48,12 +49,19 @@ public class MeasurementResolver extends
AbstractResolver<DataLakeMeasure> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
+ public void writeDocument(String document, AssetExportConfiguration config)
throws JsonProcessingException {
getNoSqlStore().getDataLakeStorage().persist(deserializeDocument(document));
}
@Override
- protected DataLakeMeasure deserializeDocument(String document) throws
JsonProcessingException {
+ public DataLakeMeasure deserializeDocument(String document) throws
JsonProcessingException {
return this.spMapper.readValue(document, DataLakeMeasure.class);
}
+
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var measurement = readDocument(document);
+ var resourceId = measurement.getElementId();
+ getNoSqlStore().getDataLakeStorage().deleteElementById(resourceId);
+ }
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
index 53830c166f..73a4404fba 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/PipelineResolver.java
@@ -19,11 +19,15 @@
package org.apache.streampipes.export.resolver;
import org.apache.streampipes.export.utils.SerializationUtils;
+import org.apache.streampipes.manager.pipeline.PipelineManager;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
import org.apache.streampipes.model.export.ExportItem;
import org.apache.streampipes.model.pipeline.Pipeline;
+import org.apache.streampipes.resource.management.secret.SecretProvider;
import com.fasterxml.jackson.core.JsonProcessingException;
+import java.util.ArrayList;
import java.util.stream.Collectors;
public class PipelineResolver extends AbstractResolver<Pipeline> {
@@ -40,9 +44,8 @@ public class PipelineResolver extends
AbstractResolver<Pipeline> {
doc.setRunning(false);
doc.setSepas(doc.getSepas().stream().peek(s ->
s.setSelectedEndpointUrl(null)).collect(Collectors.toList()));
doc.setActions(doc.getActions().stream().peek(s ->
s.setSelectedEndpointUrl(null)).collect(Collectors.toList()));
- doc.setStreams(doc.getStreams()
- .stream()
- .collect(Collectors.toList()));
+ doc.setStreams(new ArrayList<>(doc.getStreams()));
+ SecretProvider.getDecryptionService().apply(doc);
return doc;
}
@@ -57,14 +60,10 @@ public class PipelineResolver extends
AbstractResolver<Pipeline> {
}
@Override
- public void writeDocument(String document) throws JsonProcessingException {
-
getNoSqlStore().getPipelineStorageAPI().persist(deserializeDocument(document));
- }
-
public void writeDocument(String document,
- boolean overrideDocument) throws
JsonProcessingException {
+ AssetExportConfiguration config) throws
JsonProcessingException {
var pipeline = deserializeDocument(document);
- if (overrideDocument) {
+ if (config.isOverrideBrokerSettings()) {
pipeline.setSepas(pipeline.getSepas().stream().peek(processor -> {
processor.getInputStreams()
.forEach(is -> overrideProtocol(is.getEventGrounding()));
@@ -81,11 +80,25 @@ public class PipelineResolver extends
AbstractResolver<Pipeline> {
}).collect(Collectors.toList()));
}
+ SecretProvider.getEncryptionService().apply(pipeline);
getNoSqlStore().getPipelineStorageAPI().persist(pipeline);
}
@Override
- protected Pipeline deserializeDocument(String document) throws
JsonProcessingException {
+ public Pipeline deserializeDocument(String document) throws
JsonProcessingException {
return this.spMapper.readValue(document, Pipeline.class);
}
+
+ @Override
+ public void deleteDocument(String document) throws JsonProcessingException {
+ var pipeline = readDocument(document);
+ var resourceId = pipeline.getElementId();
+ var storedPipeline = PipelineManager.getPipeline(resourceId);
+ if (storedPipeline != null) {
+ if (storedPipeline.isRunning()) {
+ PipelineManager.stopPipeline(resourceId, true);
+ }
+ getNoSqlStore().getPipelineStorageAPI().deleteElementById(resourceId);
+ }
+ }
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ImportAdapterMigrationUtils.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ImportAdapterMigrationUtils.java
deleted file mode 100644
index 4100453e43..0000000000
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ImportAdapterMigrationUtils.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.streampipes.export.utils;
-
-import
org.apache.streampipes.model.connect.adapter.migration.GenericAdapterConverter;
-import
org.apache.streampipes.model.connect.adapter.migration.IAdapterConverter;
-import
org.apache.streampipes.model.connect.adapter.migration.SpecificAdapterConverter;
-import
org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels;
-
-import com.google.gson.JsonObject;
-import com.google.gson.JsonParser;
-
-import static
org.apache.streampipes.model.connect.adapter.migration.utils.AdapterModels.GENERIC_STREAM;
-
-public class ImportAdapterMigrationUtils {
-
- public static String checkAndPerformMigration(String document) {
- JsonObject doc = JsonParser.parseString(document).getAsJsonObject();
- var docType = doc.get("@class").getAsString();
- if (AdapterModels.shouldMigrate(docType)) {
- if (AdapterModels.isSetAdapter(docType)) {
- throw new IllegalArgumentException("Sets are no longer supported");
- } else {
- var converter = getAdapterConverter(docType);
- return converter.convert(doc).toString();
- }
- } else {
- return doc.toString();
- }
- }
-
- private static IAdapterConverter getAdapterConverter(String adapterType) {
- if (adapterType.equals(GENERIC_STREAM)) {
- return new GenericAdapterConverter(true);
- } else {
- return new SpecificAdapterConverter(true);
- }
- }
-}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
index bd1e946258..84c6ad3f46 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/AssetExportConfiguration.java
@@ -34,8 +34,10 @@ public class AssetExportConfiguration {
private Set<ExportItem> dataSources;
private Set<ExportItem> pipelines;
private Set<ExportItem> files;
+ private Set<ExportItem> genericStorageDocuments;
private boolean overrideBrokerSettings;
+ private boolean overwriteExistingDocuments;
public AssetExportConfiguration() {
this.adapters = new HashSet<>();
@@ -46,6 +48,7 @@ public class AssetExportConfiguration {
this.pipelines = new HashSet<>();
this.files = new HashSet<>();
this.assets = new HashSet<>();
+ this.genericStorageDocuments = new HashSet<>();
}
public Set<ExportItem> getAdapters() {
@@ -167,4 +170,24 @@ public class AssetExportConfiguration {
public void setOverrideBrokerSettings(boolean overrideBrokerSettings) {
this.overrideBrokerSettings = overrideBrokerSettings;
}
+
+ public Set<ExportItem> getGenericStorageDocuments() {
+ return genericStorageDocuments;
+ }
+
+ public void setGenericStorageDocuments(Set<ExportItem>
genericStorageDocuments) {
+ this.genericStorageDocuments = genericStorageDocuments;
+ }
+
+ public void addGenericStorageDocument(ExportItem exportItem) {
+ this.genericStorageDocuments.add(exportItem);
+ }
+
+ public boolean isOverwriteExistingDocuments() {
+ return overwriteExistingDocuments;
+ }
+
+ public void setOverwriteExistingDocuments(boolean
overwriteExistingDocuments) {
+ this.overwriteExistingDocuments = overwriteExistingDocuments;
+ }
}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
index a95dc6344a..2fee061875 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/StreamPipesApplicationPackage.java
@@ -40,6 +40,7 @@ public class StreamPipesApplicationPackage {
private Set<String> dataSources;
private Set<String> pipelines;
private Set<String> files;
+ private Set<String> genericStorageDocuments;
public StreamPipesApplicationPackage() {
this.requiredProcessorAppIds = new HashSet<>();
@@ -56,6 +57,7 @@ public class StreamPipesApplicationPackage {
this.dataSources = new HashSet<>();
this.pipelines = new HashSet<>();
this.files = new HashSet<>();
+ this.genericStorageDocuments = new HashSet<>();
}
public Set<String> getRequiredProcessorAppIds() {
@@ -201,4 +203,16 @@ public class StreamPipesApplicationPackage {
public void addAsset(String asset) {
this.assets.add(asset);
}
+
+ public Set<String> getGenericStorageDocuments() {
+ return genericStorageDocuments;
+ }
+
+ public void setGenericStorageDocuments(Set<String> genericStorageDocuments) {
+ this.genericStorageDocuments = genericStorageDocuments;
+ }
+
+ public void addGenericStorageDocument(String documentId) {
+ this.genericStorageDocuments.add(documentId);
+ }
}
diff --git
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
index 32e57df0af..ff16df6dc8 100644
---
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
+++
b/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
@@ -35,6 +35,8 @@ public interface CRUDStorage<T> {
default void deleteElementById(String id) {
var element = getElementById(id);
- deleteElement(element);
+ if (element != null) {
+ deleteElement(element);
+ }
}
}
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index c990b15011..157c3538f5 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -16,10 +16,11 @@
* specific language governing permissions and limitations
* under the License.
*/
+
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2025-02-24
11:25:01.
+// Generated using typescript-generator version 3.2.1263 on 2025-03-10
14:35:15.
export class NamedStreamPipesEntity implements Storable {
'@class':
@@ -610,7 +611,9 @@ export class AssetExportConfiguration {
dataSources: ExportItem[];
dataViews: ExportItem[];
files: ExportItem[];
+ genericStorageDocuments: ExportItem[];
overrideBrokerSettings: boolean;
+ overwriteExistingDocuments: boolean;
pipelines: ExportItem[];
static fromData(
@@ -640,7 +643,11 @@ export class AssetExportConfiguration {
data.dataViews,
);
instance.files = __getCopyArrayFn(ExportItem.fromData)(data.files);
+ instance.genericStorageDocuments = __getCopyArrayFn(
+ ExportItem.fromData,
+ )(data.genericStorageDocuments);
instance.overrideBrokerSettings = data.overrideBrokerSettings;
+ instance.overwriteExistingDocuments = data.overwriteExistingDocuments;
instance.pipelines = __getCopyArrayFn(ExportItem.fromData)(
data.pipelines,
);
@@ -3809,6 +3816,7 @@ export class StreamPipesApplicationPackage {
dataViewWidgets: string[];
dataViews: string[];
files: string[];
+ genericStorageDocuments: string[];
pipelines: string[];
requiredAdapterAppIds: string[];
requiredDataSinkAppIds: string[];
@@ -3845,6 +3853,9 @@ export class StreamPipesApplicationPackage {
data.dataViews,
);
instance.files = __getCopyArrayFn(__identity<string>())(data.files);
+ instance.genericStorageDocuments = __getCopyArrayFn(
+ __identity<string>(),
+ )(data.genericStorageDocuments);
instance.pipelines = __getCopyArrayFn(__identity<string>())(
data.pipelines,
);
diff --git a/ui/src/app/configuration/configuration.module.ts
b/ui/src/app/configuration/configuration.module.ts
index ff860383f5..accdfdec37 100644
--- a/ui/src/app/configuration/configuration.module.ts
+++ b/ui/src/app/configuration/configuration.module.ts
@@ -97,6 +97,8 @@ import { FileUploadDialogComponent } from
'./dialog/file-upload/file-upload-dial
import { FileRenameDialogComponent } from
'./dialog/file-rename/file-rename-dialog.component';
import { MatDialogModule } from '@angular/material/dialog';
import { MatProgressBarModule } from '@angular/material/progress-bar';
+import { GenericStorageItemComponent } from
'./export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component';
+import { GenericStorageItemsComponent } from
'./export/export-dialog/generic-storage-items/generic-storage-items.component';
@NgModule({
imports: [
@@ -215,6 +217,8 @@ import { MatProgressBarModule } from
'@angular/material/progress-bar';
FileUploadDialogComponent,
FileRenameDialogComponent,
GeneralConfigurationComponent,
+ GenericStorageItemComponent,
+ GenericStorageItemsComponent,
ExtensionsServiceManagementComponent,
LocationFeaturesConfigurationComponent,
ManageSiteDialogComponent,
diff --git
a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
index a088cf3333..2f23b38618 100644
---
a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
+++
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.html
@@ -54,6 +54,10 @@
[exportItems]="config.pipelines"
sectionTitle="Pipelines"
></sp-data-export-item>
+ <sp-generic-storage-items
+ [exportItems]="config.genericStorageDocuments"
+ >
+ </sp-generic-storage-items>
</div>
</div>
<div
diff --git
a/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.html
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.html
new file mode 100644
index 0000000000..78e6b097e0
--- /dev/null
+++
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.html
@@ -0,0 +1,39 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div fxLayout="row" fxLayoutAlign="start center">
+ @if (!importMode) {
+ <span fxFlex>{{ exportItem.resourceId }}</span>
+ <div fxLayoutAlign="end center">
+ <button
+ mat-icon-button
+ color="accent"
+ (click)="removeItem.emit(exportItem.resourceId)"
+ >
+ <i class="material-icons">remove</i>
+ </button>
+ </div>
+ } @else {
+ <mat-checkbox
+ [checked]="exportItem.selected"
+ (change)="removeItem.emit(exportItem.resourceId)"
+ >
+ {{ exportItem.resourceId }}
+ </mat-checkbox>
+ }
+</div>
diff --git
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.ts
similarity index 64%
copy from
streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
copy to
ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.ts
index 32e57df0af..84c2e0cfce 100644
---
a/streampipes-storage-api/src/main/java/org/apache/streampipes/storage/api/CRUDStorage.java
+++
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-item/generic-storage-item.component.ts
@@ -15,26 +15,21 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.storage.api;
-import org.apache.streampipes.model.Tuple2;
+import { Component, EventEmitter, Input, Output } from '@angular/core';
+import { ExportItem } from '@streampipes/platform-services';
-import java.util.List;
+@Component({
+ selector: 'sp-generic-storage-item',
+ templateUrl: './generic-storage-item.component.html',
+})
+export class GenericStorageItemComponent {
+ @Input()
+ exportItem: ExportItem;
-public interface CRUDStorage<T> {
+ @Input()
+ importMode = false;
- List<T> findAll();
-
- Tuple2<Boolean, String> persist(T element);
-
- T getElementById(String id);
-
- T updateElement(T element);
-
- void deleteElement(T element);
-
- default void deleteElementById(String id) {
- var element = getElementById(id);
- deleteElement(element);
- }
+ @Output()
+ removeItem: EventEmitter<string> = new EventEmitter<string>();
}
diff --git
a/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.html
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.html
new file mode 100644
index 0000000000..0d42e274c1
--- /dev/null
+++
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.html
@@ -0,0 +1,54 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div fxLayout="column" fxFlex="100">
+ <h5>Additional documents from generic storage</h5>
+
+ @for (exportItem of exportItems; track exportItem.resourceId) {
+ <sp-generic-storage-item
+ [exportItem]="exportItem"
+ [importMode]="importMode"
+ (removeItem)="handleDocumentRemoval($event)"
+ >
+ </sp-generic-storage-item>
+ }
+
+ @if (!importMode) {
+ <div fxLayout="row" fxLayoutAlign="start center" class="mt-10">
+ <div fxLayoutAlign="start center" fxFlex>
+ <mat-form-field color="accent" fxFlex
subscriptSizing="dynamic">
+ <mat-label>Document ID</mat-label>
+ <input
+ matInput
+ placeholder="Document ID"
+ [(ngModel)]="newDocumentId"
+ />
+ </mat-form-field>
+ </div>
+ <div fxLayoutAlign="end center" class="ml-5">
+ <button
+ mat-raised-button
+ color="accent"
+ (click)="addGenericStorageDocument()"
+ >
+ Add
+ </button>
+ </div>
+ </div>
+ }
+</div>
diff --git
a/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
new file mode 100644
index 0000000000..1f492681a5
--- /dev/null
+++
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
@@ -0,0 +1,36 @@
+import { Component, Input } from '@angular/core';
+import { ExportItem } from '@streampipes/platform-services';
+
+@Component({
+ selector: 'sp-generic-storage-items',
+ templateUrl: './generic-storage-items.component.html',
+})
+export class GenericStorageItemsComponent {
+ @Input()
+ exportItems: ExportItem[];
+
+ @Input()
+ importMode = false;
+
+ newDocumentId = '';
+
+ addGenericStorageDocument(): void {
+ this.exportItems.push({
+ resourceId: this.newDocumentId,
+ label: this.newDocumentId,
+ selected: true,
+ });
+ }
+
+ handleDocumentRemoval(resourceId: string): void {
+ const index = this.exportItems.findIndex(
+ e => e.resourceId === resourceId,
+ );
+ if (!this.importMode) {
+ this.exportItems.splice(index, 1);
+ } else {
+ this.exportItems[index].selected =
+ !this.exportItems[index].selected;
+ }
+ }
+}
diff --git
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
index 9e60148469..d014998557 100644
---
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
+++
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
@@ -60,11 +60,11 @@
data-cy="select-upload-file-button"
>
<mat-icon *ngIf="uploadStatus < 99"
- >insert_drive_file</mat-icon
- >
+ >insert_drive_file
+ </mat-icon>
<mat-icon *ngIf="uploadStatus === 100" class="green-icon"
- >check_circle</mat-icon
- >
+ >check_circle
+ </mat-icon>
</button>
<mat-error *ngIf="!hasInput">
{{ errorMessage }}
@@ -77,7 +77,24 @@
class="p-15"
*ngIf="currentImportStep === 1"
>
- <h4>Select resources to import</h4>
+ <div fxLayout="row" fxLayoutAlign="start center">
+ <h4>Select resources to import</h4>
+ <button
+ mat-raised-button
+ color="accent"
+ class="small-button ml-5"
+ (click)="toggleSelect(true)"
+ >
+ Select all
+ </button>
+ <button
+ mat-raised-button
+ class="small-button mat-basic"
+ (click)="toggleSelect(false)"
+ >
+ Deselect all
+ </button>
+ </div>
<sp-data-export-item
[exportItems]="importConfiguration.assets"
sectionTitle="Assets"
@@ -110,7 +127,11 @@
[exportItems]="importConfiguration.pipelines"
sectionTitle="Pipelines"
></sp-data-export-item>
-
+ <sp-generic-storage-items
+ [exportItems]="importConfiguration.genericStorageDocuments"
+ [importMode]="true"
+ >
+ </sp-generic-storage-items>
<div
fxFlex="100"
fxLayout="column"
@@ -121,8 +142,17 @@
<div fxLayout="column" *ngIf="importConfiguration">
<mat-checkbox
[(ngModel)]="importConfiguration.overrideBrokerSettings"
- >Use broker settings from this instance</mat-checkbox
- >
+ >Use broker settings from this instance
+ </mat-checkbox>
+ <div class="warning-box">
+ <mat-checkbox
+ [(ngModel)]="
+ importConfiguration.overwriteExistingDocuments
+ "
+ >Update/overwrite existing documents with the same
+ ID (operation may break things)
+ </mat-checkbox>
+ </div>
</div>
</div>
</div>
diff --git
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
index fddade7bf6..2251ca4f01 100644
---
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
+++
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.scss
@@ -17,3 +17,8 @@
*/
@import '../../../../scss/sp/sp-dialog.scss';
+
+.warning-box {
+ border: 1px solid #dea843;
+ background: #f1f1e6;
+}
diff --git
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
index 05e341410e..5f23b5d1d7 100644
---
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
+++
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
@@ -20,7 +20,10 @@ import { Component } from '@angular/core';
import { DialogRef } from '@streampipes/shared-ui';
import { DataExportService } from '../data-export.service';
import { HttpEventType, HttpResponse } from '@angular/common/http';
-import { AssetExportConfiguration } from
'../../../../../dist/streampipes/platform-services';
+import {
+ AssetExportConfiguration,
+ ExportItem,
+} from '@streampipes/platform-services';
@Component({
selector: 'sp-data-import-dialog',
@@ -93,4 +96,29 @@ export class SpDataImportDialogComponent {
close(): void {
this.dialogRef.close();
}
+
+ toggleSelect(select: boolean): void {
+ if (this.importConfiguration) {
+ this.toggleExportItems(select);
+ }
+ }
+
+ private toggleExportItems(select: boolean): void {
+ this.toggleAllItems(this.importConfiguration.files, select);
+ this.toggleAllItems(this.importConfiguration.dataSources, select);
+ this.toggleAllItems(this.importConfiguration.adapters, select);
+ this.toggleAllItems(this.importConfiguration.assets, select);
+ this.toggleAllItems(this.importConfiguration.dashboards, select);
+ this.toggleAllItems(this.importConfiguration.dataViews, select);
+ this.toggleAllItems(this.importConfiguration.dataLakeMeasures, select);
+ this.toggleAllItems(
+ this.importConfiguration.genericStorageDocuments,
+ select,
+ );
+ this.toggleAllItems(this.importConfiguration.pipelines, select);
+ }
+
+ private toggleAllItems(exportItem: ExportItem[], select: boolean): void {
+ exportItem.forEach(e => (e.selected = select));
+ }
}
