This is an automated email from the ASF dual-hosted git repository.
dominikriemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new e321304ef5 feat: Improve handling of generic storage document for
imports and exports (#5154)
e321304ef5 is described below
commit e321304ef5942caad2bb5745a24dbf395afe5e06
Author: Sven Oehler <[email protected]>
AuthorDate: Fri Oct 30 10:12:27 2026 +0100
feat: Improve handling of generic storage document for imports and exports
(#5154)
Co-authored-by: Dominik Riemer <[email protected]>
---
streampipes-data-export/pom.xml | 10 +
.../streampipes/export/AssetLinkResolver.java | 6 +-
.../apache/streampipes/export/ExportManager.java | 42 ++-
.../export/GenericStorageResourceRegistry.java | 45 +++
.../streampipes/export/PermissionDefaults.java | 41 +--
.../streampipes/export/ResourceRegistration.java | 40 +++
.../export/dataimport/PerformImportGenerator.java | 30 +-
.../export/dataimport/PreviewImportGenerator.java | 10 +-
.../export/generator/ExportPackageGenerator.java | 53 ++--
.../export/resolver/AbstractResolver.java | 6 +-
.../resolver/GenericStorageDocumentResolver.java | 2 +-
.../resolver/GenericStorageResolverProvider.java | 106 +++++++
.../export/utils/ArchiveEntryNames.java | 40 +--
.../streampipes/export/AssetLinkResolverTest.java | 65 ++++
.../export/GenericStorageCategoryExportTest.java | 333 +++++++++++++++++++++
.../export/GenericStorageResolverTest.java | 143 +++++++++
.../export/GenericStorageResourceRegistryTest.java | 109 +++++++
.../model/export/ExportConfiguration.java | 10 +
.../streampipes/model/export/ExportItem.java | 9 +
.../src/lib/model/gen/streampipes-model.ts | 8 +-
.../assets/dialog/move-asset-dialog.component.ts | 2 -
.../export/data-export-import.component.ts | 2 +
.../data-export-dialog.component.spec.ts | 123 ++++++++
.../export-dialog/data-export-dialog.component.ts | 78 ++++-
.../generic-storage-items.component.ts | 1 +
.../data-import-dialog.component.html | 22 +-
.../import-dialog/data-import-dialog.component.ts | 10 +
27 files changed, 1216 insertions(+), 130 deletions(-)
diff --git a/streampipes-data-export/pom.xml b/streampipes-data-export/pom.xml
index 623f074903..a9d89132fb 100644
--- a/streampipes-data-export/pom.xml
+++ b/streampipes-data-export/pom.xml
@@ -61,6 +61,16 @@
<groupId>org.lightcouch</groupId>
<artifactId>lightcouch</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.junit.jupiter</groupId>
+ <artifactId>junit-jupiter-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
index af8aa370d1..5b120a6860 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/AssetLinkResolver.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.export.resolver.ChartResolver;
import org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
import
org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
@@ -100,6 +101,9 @@ public class AssetLinkResolver {
exportConfig.setFiles(new
FileResolver(resourceManager.getFileMetadataStorage())
.resolve(getLinks(assetLinks, ResolvableAssetLinks.FILE)));
+ exportConfig.setGenericStorageDocuments(
+ new
GenericStorageResolverProvider(resourceManager.getGenericStorage()).resolve(assetLinks));
+
return exportConfig;
} catch (IOException e) {
LOG.error("IO Exception when writing export configuration", e);
@@ -111,7 +115,7 @@ public class AssetLinkResolver {
String queryHint) {
return assetLinks
.stream()
- .filter(link -> link.getQueryHint().equals(queryHint))
+ .filter(link -> queryHint.equals(link.getQueryHint()))
.collect(Collectors.toSet());
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
index 88c6d4f2d6..35b791fef7 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ExportManager.java
@@ -19,6 +19,7 @@
package org.apache.streampipes.export;
import org.apache.streampipes.export.generator.ExportPackageGenerator;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
import
org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
import org.apache.streampipes.manager.pipeline.PipelineManager;
import org.apache.streampipes.model.assets.SpAssetModel;
@@ -28,6 +29,7 @@ import
org.apache.streampipes.resource.management.SpResourceManager;
import org.apache.streampipes.storage.api.system.IGenericStorage;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -44,10 +46,8 @@ public class ExportManager {
assetId, extensionServiceRequestManager, resourceManager,
pipelineManager)
.resolveResources())
.collect(Collectors.toList());
- var genericStorageAppDocTypes =
getGenericStorageAppDocTypes(resourceManager.getGenericStorage());
-
exportConfig.setAssetExportConfiguration(assetExportConfigurations);
- exportConfig.setGenericStorageAppDocTypes(genericStorageAppDocTypes);
+ addGenericStoragePreview(exportConfig,
resourceManager.getGenericStorage(), selectedAssetIds);
return exportConfig;
}
@@ -61,16 +61,36 @@ public class ExportManager {
.generateExportPackage();
}
- private static List<ExportItem> getGenericStorageAppDocTypes(IGenericStorage
genericStorage) {
+ private static void addGenericStoragePreview(ExportConfiguration config,
+ IGenericStorage storage,
+ List<String> selectedAssetIds)
{
+ var resolvers = new GenericStorageResolverProvider(storage);
try {
- return genericStorage
- .getAllAppDocTypes()
- .stream()
- .filter(appDocType -> !SpAssetModel.APP_DOC_TYPE.equals(appDocType))
- .map(appDocType -> new ExportItem(appDocType, appDocType, false))
- .collect(Collectors.toList());
+ for (var appDocType : storage.getAllAppDocTypes()) {
+ if (SpAssetModel.APP_DOC_TYPE.equals(appDocType)) {
+ continue;
+ }
+ var exportType =
GenericStorageResourceRegistry.INSTANCE.find(appDocType).filter(type ->
type.category() != null);
+ if (exportType.isPresent()) {
+ if (!selectedAssetIds.isEmpty()) {
+ continue;
+ }
+ // Global exports list all documents; asset exports are resolved by
AssetLinkResolver.
+ try {
+ var items = new ArrayList<ExportItem>();
+ for (var document : storage.findAll(appDocType)) {
+ items.add(resolvers.toExportItem(document, false));
+ }
+ config.getGenericStorageDocuments().addAll(items);
+ } catch (IOException e) {
+ config.addGenericStorageAppDocType(new ExportItem(appDocType,
appDocType, false));
+ }
+ } else {
+ config.addGenericStorageAppDocType(new ExportItem(appDocType,
appDocType, false));
+ }
+ }
} catch (IOException e) {
- return List.of();
+ // Preserve the asset preview if generic storage is unavailable.
}
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/GenericStorageResourceRegistry.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/GenericStorageResourceRegistry.java
new file mode 100644
index 0000000000..41376ca240
--- /dev/null
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/GenericStorageResourceRegistry.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** Registrations for resource types persisted in generic storage. */
+public class GenericStorageResourceRegistry {
+
+ public static final GenericStorageResourceRegistry INSTANCE = new
GenericStorageResourceRegistry();
+
+ private final Map<String, ResourceRegistration<?>> registrations = new
ConcurrentHashMap<>();
+
+ public void register(String appDocType, ResourceRegistration<?>
registration) {
+ Objects.requireNonNull(appDocType, "appDocType");
+ Objects.requireNonNull(registration, "registration");
+ var existing = registrations.putIfAbsent(appDocType, registration);
+ if (existing != null && !existing.equals(registration)) {
+ throw new IllegalArgumentException("Resource type already registered for
" + appDocType);
+ }
+ }
+
+ public Optional<ResourceRegistration<?>> find(String appDocType) {
+ return Optional.ofNullable(registrations.get(appDocType));
+ }
+}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/PermissionDefaults.java
similarity index 52%
copy from
streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
copy to
streampipes-data-export/src/main/java/org/apache/streampipes/export/PermissionDefaults.java
index cb62711a05..ac630aa56a 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/PermissionDefaults.java
@@ -16,44 +16,13 @@
*
*/
-package org.apache.streampipes.model.export;
+package org.apache.streampipes.export;
-public class ExportItem {
+import java.util.Objects;
- private String resourceId;
- private String label;
- private boolean selected;
+public record PermissionDefaults(Class<?> resourceClass, boolean
publicElement) {
- public ExportItem() {
- }
-
- public ExportItem(String resourceId, String label, boolean selected) {
- this.resourceId = resourceId;
- this.label = label;
- this.selected = selected;
- }
-
- public String getResourceId() {
- return resourceId;
- }
-
- public void setResourceId(String resourceId) {
- this.resourceId = resourceId;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public boolean isSelected() {
- return selected;
- }
-
- public void setSelected(boolean selected) {
- this.selected = selected;
+ public PermissionDefaults {
+ Objects.requireNonNull(resourceClass, "resourceClass");
}
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/ResourceRegistration.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ResourceRegistration.java
new file mode 100644
index 0000000000..25811ffbbd
--- /dev/null
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/ResourceRegistration.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.export.resolver.AbstractResolver;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import java.util.Objects;
+import java.util.function.Function;
+
+/** Metadata and a required document resolver factory for an external resource
type. */
+public record ResourceRegistration<T>(PermissionDefaults permissions,
+ String category,
+ Function<IGenericStorage, ? extends
AbstractResolver<T>> resolverFactory) {
+
+ public ResourceRegistration {
+ Objects.requireNonNull(permissions, "permissions");
+ Objects.requireNonNull(resolverFactory, "resolverFactory");
+ }
+
+ public AbstractResolver<T> createResolver(IGenericStorage storage) {
+ return Objects.requireNonNull(resolverFactory.apply(storage), "resolver");
+ }
+}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
index c225b2956f..7fe960e711 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PerformImportGenerator.java
@@ -18,6 +18,7 @@
package org.apache.streampipes.export.dataimport;
+import org.apache.streampipes.export.GenericStorageResourceRegistry;
import org.apache.streampipes.export.model.PermissionInfo;
import org.apache.streampipes.export.resolver.AbstractResolver;
import org.apache.streampipes.export.resolver.AdapterResolver;
@@ -26,8 +27,10 @@ import
org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
+import org.apache.streampipes.export.utils.ArchiveEntryNames;
import
org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
import org.apache.streampipes.manager.file.FileHandler;
import org.apache.streampipes.manager.pipeline.PipelineManager;
@@ -53,6 +56,7 @@ import java.util.Set;
public class PerformImportGenerator extends ImportGenerator<Void> {
+ private final GenericStorageResolverProvider genericStorageResolvers;
private final AssetExportConfiguration config;
private final IGenericStorage genericStorage;
private final Set<PermissionInfo> permissionsToStore = new HashSet<>();
@@ -67,8 +71,19 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
ExtensionServiceRequestManager
extensionServiceRequestManager,
SpResourceManager resourceManager,
PipelineManager pipelineManager) {
+ this(config, ownerSid, extensionServiceRequestManager, resourceManager,
pipelineManager,
+ GenericStorageResourceRegistry.INSTANCE);
+ }
+
+ public PerformImportGenerator(AssetExportConfiguration config,
+ String ownerSid,
+ ExtensionServiceRequestManager
extensionServiceRequestManager,
+ SpResourceManager resourceManager,
+ PipelineManager pipelineManager,
+ GenericStorageResourceRegistry
resourceRegistry) {
this.config = config;
this.genericStorage = resourceManager.getGenericStorage();
+ this.genericStorageResolvers = new
GenericStorageResolverProvider(genericStorage, resourceRegistry);
this.ownerSid = ownerSid;
this.extensionServiceRequestManager = extensionServiceRequestManager;
this.resourceManager = resourceManager;
@@ -175,7 +190,20 @@ public class PerformImportGenerator extends
ImportGenerator<Void> {
@Override
protected void handleGenericStorageDocument(String document, String
documentId) throws JsonProcessingException {
if (shouldStore(documentId, config.getGenericStorageDocuments())) {
- writeDocument(document, new
GenericStorageDocumentResolver(genericStorage));
+ var resolver = new GenericStorageDocumentResolver(genericStorage);
+ var storedDocument = resolver.readDocument(document);
+ if (!(storedDocument.get("_id") instanceof String resourceId)
+ || !(documentId.equals(resourceId) ||
documentId.equals(ArchiveEntryNames.forResourceId(resourceId)))) {
+ throw new IllegalArgumentException("Generic document id does not match
manifest: " + documentId);
+ }
+ writeDocument(document,
genericStorageResolvers.resolverFor(storedDocument));
+
genericStorageResolvers.registrationFor(storedDocument).ifPresent(registration
-> {
+ var permissions = registration.permissions();
+ if
(resourceManager.managePermissions().findForObjectId(resourceId).isEmpty()) {
+ resourceManager.managePermissions().createDefault(resourceId,
+ permissions.resourceClass(), ownerSid,
permissions.publicElement());
+ }
+ });
}
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
index 6ba6c937ed..2328c5a9dd 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/dataimport/PreviewImportGenerator.java
@@ -24,6 +24,7 @@ import org.apache.streampipes.export.resolver.ChartResolver;
import org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
import
org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
@@ -151,8 +152,13 @@ public class PreviewImportGenerator extends
ImportGenerator<AssetExportConfigura
}
@Override
- protected void handleGenericStorageDocument(String document, String
genericDocId) {
- addExportItem(genericDocId, genericDocId,
importConfig::addGenericStorageDocument);
+ protected void handleGenericStorageDocument(String document, String
genericDocId) throws JsonProcessingException {
+ Map<String, Object> payload = defaultMapper.readValue(document, new
TypeReference<>() { });
+ var resolvers = new
GenericStorageResolverProvider(resourceManager.getGenericStorage());
+ var item = resolvers.toExportItem(payload, true);
+ // Import selections identify archive entries, whose names may differ from
stored document IDs.
+ item.setResourceId(genericDocId);
+ importConfig.addGenericStorageDocument(item);
}
@Override
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
index 8709874696..6352b0ae9d 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/generator/ExportPackageGenerator.java
@@ -25,9 +25,10 @@ import org.apache.streampipes.export.resolver.ChartResolver;
import org.apache.streampipes.export.resolver.DashboardResolver;
import org.apache.streampipes.export.resolver.DataSourceResolver;
import org.apache.streampipes.export.resolver.FileResolver;
-import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
import org.apache.streampipes.export.resolver.MeasurementResolver;
import org.apache.streampipes.export.resolver.PipelineResolver;
+import org.apache.streampipes.export.utils.ArchiveEntryNames;
import
org.apache.streampipes.manager.api.extensions.ExtensionServiceRequestManager;
import org.apache.streampipes.manager.file.FileManager;
import org.apache.streampipes.manager.pipeline.PipelineManager;
@@ -63,6 +64,7 @@ public class ExportPackageGenerator {
private final ObjectMapper defaultMapper;
private final PipelineManager pipelineManager;
private final SpResourceManager resourceManager;
+ private final GenericStorageResolverProvider genericStorageResolvers;
public ExportPackageGenerator(ExportConfiguration exportConfiguration,
ExtensionServiceRequestManager
extensionServiceRequestManager,
@@ -72,6 +74,7 @@ public class ExportPackageGenerator {
this.extensionServiceRequestManager = extensionServiceRequestManager;
this.pipelineManager = pipelineManager;
this.resourceManager = resourceManager;
+ this.genericStorageResolvers = new
GenericStorageResolverProvider(resourceManager.getGenericStorage());
this.defaultMapper = JacksonSerializer.getObjectMapper(Map.of(
DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true,
SerializationFeature.INDENT_OUTPUT, false
@@ -136,17 +139,16 @@ public class ExportPackageGenerator {
config.getGenericStorageDocuments().forEach(item -> {
addGenericStorageDocument(builder,
item,
- new
GenericStorageDocumentResolver(resourceManager.getGenericStorage()),
manifest::addGenericStorageDocument,
exportedGenericStorageDocumentIds);
});
config.getLabels().forEach(item -> {
- addDoc(builder, item, new
GenericStorageDocumentResolver(resourceManager.getGenericStorage()),
manifest::addGenericStorageDocument);
+ addGenericStorageDocument(builder, item,
manifest::addGenericStorageDocument, exportedGenericStorageDocumentIds);
});
config.getSites().forEach(item -> {
- addDoc(builder, item, new
GenericStorageDocumentResolver(resourceManager.getGenericStorage()),
manifest::addGenericStorageDocument);
+ addGenericStorageDocument(builder, item,
manifest::addGenericStorageDocument, exportedGenericStorageDocumentIds);
});
config.getFiles().forEach(item -> {
@@ -168,6 +170,13 @@ public class ExportPackageGenerator {
});
});
+ if (exportConfiguration.getGenericStorageDocuments() != null) {
+ exportConfiguration.getGenericStorageDocuments().forEach(item ->
addGenericStorageDocument(builder,
+ item,
+ manifest::addGenericStorageDocument,
+ exportedGenericStorageDocumentIds));
+ }
+
if (exportConfiguration.getGenericStorageAppDocTypes() != null) {
exportConfiguration.getGenericStorageAppDocTypes().forEach(item -> {
if (item.isSelected()) {
@@ -196,34 +205,39 @@ public class ExportPackageGenerator {
ExportItem exportItem,
AbstractResolver<?> resolver,
Consumer<String> function) {
+ addDoc(builder, exportItem, resolver::getSerializedDocument, function);
+ }
+
+ private void addDoc(ZipFileBuilder builder,
+ ExportItem exportItem,
+ DocumentExporter exporter,
+ Consumer<String> function) {
try {
if (exportItem.isSelected()) {
var resourceId = exportItem.getResourceId();
- var sanitizedResourceId = sanitize(resourceId);
- builder.addText(sanitizedResourceId,
resolver.getSerializedDocument(resourceId));
+ var sanitizedResourceId = ArchiveEntryNames.forResourceId(resourceId);
+ builder.addText(sanitizedResourceId, exporter.serialize(resourceId));
function.accept(sanitizedResourceId);
}
} catch (JsonProcessingException | ElementNotFoundException e) {
- LOG.warn(
- "Could not find document with resource id {} with resolver {}",
- exportItem.getResourceId(),
- resolver.getClass().getCanonicalName(),
- e);
+ LOG.warn("Could not export document with resource id {}",
exportItem.getResourceId(), e);
}
}
+ @FunctionalInterface
+ private interface DocumentExporter {
+ String serialize(String resourceId) throws JsonProcessingException,
ElementNotFoundException;
+ }
+
private void addGenericStorageDocumentsByType(ZipFileBuilder builder,
String appDocType,
Consumer<String> function,
Set<String>
exportedGenericStorageDocumentIds) {
- var resolver = new
GenericStorageDocumentResolver(resourceManager.getGenericStorage());
-
try {
resourceManager.getGenericStorage()
.findAll(appDocType)
.forEach(document -> addGenericStorageDocument(builder,
- resolver.convert(document),
- resolver,
+ new ExportItem(document.get("_id").toString(), "", true),
function,
exportedGenericStorageDocumentIds));
} catch (IOException e) {
@@ -233,18 +247,13 @@ public class ExportPackageGenerator {
private void addGenericStorageDocument(ZipFileBuilder builder,
ExportItem exportItem,
- AbstractResolver<?> resolver,
Consumer<String> function,
Set<String>
exportedGenericStorageDocumentIds) {
- if (exportedGenericStorageDocumentIds.add(exportItem.getResourceId())) {
- addDoc(builder, exportItem, resolver, function);
+ if (exportItem.isSelected() &&
exportedGenericStorageDocumentIds.add(exportItem.getResourceId())) {
+ addDoc(builder, exportItem,
genericStorageResolvers::getSerializedDocument, function);
}
}
- private String sanitize(String resourceId) {
- return resourceId.replaceAll(":", "").replaceAll("\\.", "");
- }
-
private void addAssets(ZipFileBuilder builder,
List<String> assetIds,
StreamPipesApplicationPackage manifest) {
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
index 977d64bb3b..499c1ef820 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/AbstractResolver.java
@@ -62,12 +62,16 @@ public abstract class AbstractResolver<T> implements
DocumentResolver<T> {
public String getSerializedDocument(String resourceId) throws
JsonProcessingException, ElementNotFoundException {
var document = findDocument(resourceId);
if (document != null) {
- return
getObjectMapper().writeValueAsString(modifyDocumentForExport(document));
+ return serializeDocument(modifyDocumentForExport(document));
} else {
throw new ElementNotFoundException("Could not find element with resource
id " + resourceId);
}
}
+ public String serializeDocument(T document) throws JsonProcessingException {
+ return getObjectMapper().writeValueAsString(document);
+ }
+
protected void overrideProtocol(EventGrounding grounding) {
var newProtocol = new
EventGroundingProcessor().applyOverride(grounding.getTransportProtocol());
grounding.setTransportProtocol(newProtocol);
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
index e3530ecb27..beb00bfe96 100644
---
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageDocumentResolver.java
@@ -80,7 +80,7 @@ public class GenericStorageDocumentResolver extends
AbstractResolver<Map<String,
try {
genericStorage.create(document);
} catch (IOException e) {
- LOG.warn("Could not write document");
+ LOG.warn("Could not write document", e);
}
}
diff --git
a/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageResolverProvider.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageResolverProvider.java
new file mode 100644
index 0000000000..37777004df
--- /dev/null
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/resolver/GenericStorageResolverProvider.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export.resolver;
+
+import org.apache.streampipes.commons.exceptions.ElementNotFoundException;
+import org.apache.streampipes.export.GenericStorageResourceRegistry;
+import org.apache.streampipes.export.ResourceRegistration;
+import org.apache.streampipes.model.assets.AssetLink;
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/** Selects resource resolvers and bridges generic storage documents to their
domain types. */
+public class GenericStorageResolverProvider {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(GenericStorageResolverProvider.class);
+
+ private final GenericStorageResourceRegistry registry;
+ private final IGenericStorage storage;
+ private final GenericStorageDocumentResolver fallback;
+ private final ObjectMapper mapper = JacksonSerializer.getObjectMapper();
+
+ public GenericStorageResolverProvider(IGenericStorage storage) {
+ this(storage, GenericStorageResourceRegistry.INSTANCE);
+ }
+
+ public GenericStorageResolverProvider(IGenericStorage storage,
GenericStorageResourceRegistry registry) {
+ this.storage = storage;
+ this.registry = registry;
+ this.fallback = new GenericStorageDocumentResolver(storage);
+ }
+
+ public Optional<ResourceRegistration<?>> registrationFor(Map<String, Object>
document) {
+ return document.get("appDocType") instanceof String type ?
registry.find(type) : Optional.empty();
+ }
+
+ public AbstractResolver<?> resolverFor(Map<String, Object> document) {
+ return registrationFor(document).<AbstractResolver<?>>map(entry ->
entry.createResolver(storage)).orElse(fallback);
+ }
+
+ public ExportItem toExportItem(Map<String, Object> document, boolean
selected) throws JsonProcessingException {
+ var item = convert(resolverFor(document),
mapper.writeValueAsString(document));
+ item.setSelected(selected);
+ registrationFor(document).ifPresent(entry ->
item.setCategory(entry.category()));
+ return item;
+ }
+
+ public Set<ExportItem> resolve(Set<AssetLink> links) {
+ var items = new HashSet<ExportItem>();
+ var resourceIds =
links.stream().map(AssetLink::getResourceId).filter(Objects::nonNull).distinct().toList();
+ for (var resourceId : resourceIds) {
+ var document = fallback.findDocument(resourceId);
+ if (document != null && registrationFor(document).filter(entry ->
entry.category() != null).isPresent()) {
+ try {
+ items.add(toExportItem(document, true));
+ } catch (JsonProcessingException e) {
+ LOG.warn("Could not resolve linked generic storage document {}",
resourceId, e);
+ }
+ }
+ }
+ return items;
+ }
+
+ public String getSerializedDocument(String resourceId) throws
JsonProcessingException, ElementNotFoundException {
+ var document = fallback.findDocument(resourceId);
+ if (document == null) {
+ throw new ElementNotFoundException("Could not find element with resource
id " + resourceId);
+ }
+ return serializeForExport(resolverFor(document),
mapper.writeValueAsString(document));
+ }
+
+ private <T> ExportItem convert(AbstractResolver<T> resolver, String
document) throws JsonProcessingException {
+ return resolver.convert(resolver.readDocument(document));
+ }
+
+ private <T> String serializeForExport(AbstractResolver<T> resolver, String
document) throws JsonProcessingException {
+ return
resolver.serializeDocument(resolver.modifyDocumentForExport(resolver.readDocument(document)));
+ }
+}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ArchiveEntryNames.java
similarity index 52%
copy from
streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
copy to
streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ArchiveEntryNames.java
index cb62711a05..6b4d889c5d 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
+++
b/streampipes-data-export/src/main/java/org/apache/streampipes/export/utils/ArchiveEntryNames.java
@@ -16,44 +16,14 @@
*
*/
-package org.apache.streampipes.model.export;
+package org.apache.streampipes.export.utils;
-public class ExportItem {
+public final class ArchiveEntryNames {
- private String resourceId;
- private String label;
- private boolean selected;
-
- public ExportItem() {
- }
-
- public ExportItem(String resourceId, String label, boolean selected) {
- this.resourceId = resourceId;
- this.label = label;
- this.selected = selected;
- }
-
- public String getResourceId() {
- return resourceId;
- }
-
- public void setResourceId(String resourceId) {
- this.resourceId = resourceId;
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public boolean isSelected() {
- return selected;
+ private ArchiveEntryNames() {
}
- public void setSelected(boolean selected) {
- this.selected = selected;
+ public static String forResourceId(String resourceId) {
+ return resourceId.replace(":", "").replace(".", "");
}
}
diff --git
a/streampipes-data-export/src/test/java/org/apache/streampipes/export/AssetLinkResolverTest.java
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/AssetLinkResolverTest.java
new file mode 100644
index 0000000000..772c4bf957
--- /dev/null
+++
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/AssetLinkResolverTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.model.file.FileMetadata;
+import org.apache.streampipes.resource.management.SpResourceManager;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class AssetLinkResolverTest {
+
+ @Test
+ void shouldIgnoreLinksWithoutQueryHintsAndResolveSupportedLinks() throws
IOException {
+ var resources = mock(SpResourceManager.class, RETURNS_DEEP_STUBS);
+ when(resources.getGenericStorage().findOne("asset-1")).thenReturn(Map.of(
+ "assetName", "Test asset",
+ "assets", List.of(),
+ "assetLinks", List.of(
+ Map.of("resourceId", "external-link"),
+ Map.of("resourceId", "file-1", "queryHint", "file"))));
+ var file = new FileMetadata();
+ file.setFileId("file-1");
+ file.setFilename("test.csv");
+
when(resources.getFileMetadataStorage().getElementById("file-1")).thenReturn(file);
+
+ var result = new AssetLinkResolver("asset-1", null, resources,
null).resolveResources();
+
+ assertEquals("asset-1", result.getAssetId());
+ assertEquals("Test asset", result.getAssetName());
+ assertEquals(1, result.getFiles().size());
+ assertEquals("file-1",
result.getFiles().iterator().next().getResourceId());
+ assertTrue(result.getAdapters().isEmpty());
+ assertTrue(result.getDataViews().isEmpty());
+ assertTrue(result.getDashboards().isEmpty());
+ assertTrue(result.getDataSources().isEmpty());
+ assertTrue(result.getPipelines().isEmpty());
+ assertTrue(result.getDataLakeMeasures().isEmpty());
+ }
+}
diff --git
a/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageCategoryExportTest.java
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageCategoryExportTest.java
new file mode 100644
index 0000000000..94c7322023
--- /dev/null
+++
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageCategoryExportTest.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.commons.zip.ZipFileExtractor;
+import org.apache.streampipes.export.dataimport.PerformImportGenerator;
+import org.apache.streampipes.export.dataimport.PreviewImportGenerator;
+import org.apache.streampipes.export.generator.ZipFileBuilder;
+import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.model.export.ExportConfiguration;
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.model.export.StreamPipesApplicationPackage;
+import org.apache.streampipes.resource.management.SpResourceManager;
+import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class GenericStorageCategoryExportTest {
+
+ private static final String TYPE = "test:categorized-pipeline";
+ private static final ResourceRegistration<Map<String, Object>> REGISTRATION
= new ResourceRegistration<>(
+ new PermissionDefaults(GenericStorageCategoryExportTest.class, false),
+ "Test pipelines", NamedDocumentResolver::new);
+ private final IGenericStorage storage = mock(IGenericStorage.class);
+ private final SpResourceManager resources = mock(SpResourceManager.class,
RETURNS_DEEP_STUBS);
+ private final Map<String, Object> first = Map.of(
+ "_id", "pipeline-1", "appDocType", TYPE, "name", "Named pipeline",
"_rev", "1-revision");
+ private final Map<String, Object> second = Map.of("_id", "pipeline-2",
"appDocType", TYPE);
+
+ @BeforeEach
+ void setUp() throws IOException {
+ GenericStorageResourceRegistry.INSTANCE.register(TYPE, REGISTRATION);
+ when(resources.getGenericStorage()).thenReturn(storage);
+ when(storage.getAllAppDocTypes()).thenReturn(List.of(TYPE,
"other-documents"));
+ when(storage.findAll(TYPE)).thenReturn(List.of(first, second));
+ // Export sanitization removes _rev from the retrieved map.
+ when(storage.findOne("pipeline-1")).thenAnswer(invocation -> new
HashMap<>(first));
+ when(storage.findOne("pipeline-2")).thenAnswer(invocation -> new
HashMap<>(second));
+ }
+
+ @Test
+ void shouldListNamedCategorySeparatelyAndUseIdWhenNameIsMissing() {
+ var preview = preview();
+
+ assertEquals(List.of("other-documents"),
preview.getGenericStorageAppDocTypes().stream()
+ .map(ExportItem::getResourceId).toList());
+ assertEquals(List.of("Named pipeline", "pipeline-2"),
preview.getGenericStorageDocuments().stream()
+ .map(ExportItem::getLabel).toList());
+ assertTrue(preview.getGenericStorageDocuments().stream().allMatch(item ->
"Test pipelines".equals(item.getCategory())));
+
assertTrue(preview.getGenericStorageDocuments().stream().noneMatch(ExportItem::isSelected));
+ }
+
+ @Test
+ void shouldExportOnlySelectedDocumentsUsingExistingArchiveFormat() throws
IOException {
+ var preview = preview();
+ preview.getGenericStorageDocuments().getFirst().setSelected(true);
+
+ var files = new ZipFileExtractor(new
ByteArrayInputStream(export(preview))).extractZipToMap();
+ var manifest =
JacksonSerializer.getObjectMapper().readValue(files.get("manifest"),
+ StreamPipesApplicationPackage.class);
+
+ assertEquals(Set.of("pipeline-1"), manifest.getGenericStorageDocuments());
+ assertTrue(files.containsKey("pipeline-1"));
+ assertFalse(files.containsKey("pipeline-2"));
+ }
+
+ @Test
+ void shouldStillSupportBulkExportsFromOlderClients() throws IOException {
+ var config = new ExportConfiguration();
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+
+ var files = new ZipFileExtractor(new
ByteArrayInputStream(export(config))).extractZipToMap();
+ var manifest =
JacksonSerializer.getObjectMapper().readValue(files.get("manifest"),
+ StreamPipesApplicationPackage.class);
+
+ assertEquals(Set.of("pipeline-1", "pipeline-2"),
manifest.getGenericStorageDocuments());
+ }
+
+ @Test
+ void shouldCategorizeLegacyArchiveDocumentsDuringImportPreview() throws
IOException {
+ var config = new ExportConfiguration();
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+
+ var preview = new PreviewImportGenerator(null, resources, null)
+ .generate(new ByteArrayInputStream(export(config)));
+
+ assertEquals(2, preview.getGenericStorageDocuments().size());
+ assertTrue(preview.getGenericStorageDocuments().stream().allMatch(item ->
"Test pipelines".equals(item.getCategory())));
+
assertTrue(preview.getGenericStorageDocuments().stream().allMatch(ExportItem::isSelected));
+ assertTrue(preview.getGenericStorageDocuments().stream().anyMatch(item ->
"Named pipeline".equals(item.getLabel())));
+ }
+
+ @Test
+ void shouldKeepUnregisteredDocumentsUncategorized() throws IOException {
+ when(storage.findAll("other-documents")).thenReturn(List.of(Map.of(
+ "_id", "other-1", "appDocType", "other-documents", "name", "Other")));
+ when(storage.findOne("other-1")).thenAnswer(invocation -> new
HashMap<>(Map.of(
+ "_id", "other-1", "appDocType", "other-documents", "name", "Other")));
+ var config = new ExportConfiguration();
+ config.addGenericStorageAppDocType(new ExportItem("other-documents",
"Other", true));
+
+ var preview = new PreviewImportGenerator(null, resources, null)
+ .generate(new ByteArrayInputStream(export(config)));
+
+ var item = preview.getGenericStorageDocuments().iterator().next();
+ assertNull(item.getCategory());
+ assertEquals("other-1", item.getLabel());
+ }
+
+ @Test
+ void shouldFallBackToBulkSelectionWhenDocumentEnumerationFails() throws
IOException {
+ when(storage.findAll(TYPE)).thenThrow(new IOException("unavailable"));
+
+ var preview = preview();
+
+ assertTrue(preview.getGenericStorageDocuments().isEmpty());
+ assertTrue(preview.getGenericStorageAppDocTypes().stream().anyMatch(item
-> TYPE.equals(item.getResourceId())));
+ }
+
+ @Test
+ void shouldNotLetUnselectedCategoryItemsSuppressAnExplicitBulkExport()
throws IOException {
+ var config = preview();
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+
+ var files = new ZipFileExtractor(new
ByteArrayInputStream(export(config))).extractZipToMap();
+ var manifest =
JacksonSerializer.getObjectMapper().readValue(files.get("manifest"),
+ StreamPipesApplicationPackage.class);
+
+ assertEquals(Set.of("pipeline-1", "pipeline-2"),
manifest.getGenericStorageDocuments());
+ }
+
+ @Test
+ void shouldRespectIndividualSelectionsWhenImportingCategorizedDocuments()
throws IOException {
+ var config = new ExportConfiguration();
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+ var archive = export(config);
+ var importConfig = new PreviewImportGenerator(null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+ importConfig.getGenericStorageDocuments().stream()
+ .filter(item -> "pipeline-1".equals(item.getResourceId()))
+ .forEach(item -> item.setSelected(false));
+
+ new PerformImportGenerator(importConfig, "importer", null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+
+ var document = ArgumentCaptor.forClass(String.class);
+ verify(storage).create(document.capture());
+ assertEquals("pipeline-2",
JacksonSerializer.getObjectMapper().readTree(document.getValue()).get("_id").asText());
+ }
+
+ @Test
+ void shouldRoundTripDocumentsWithSanitizedArchiveIds() throws IOException {
+
when(resources.managePermissions().findForObjectId(anyString())).thenReturn(List.of());
+ var archive = exportDocumentsWithSpecialIds();
+ var importConfig = new PreviewImportGenerator(null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+
+ new PerformImportGenerator(importConfig, "importer", null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+
+ var document = ArgumentCaptor.forClass(String.class);
+ verify(storage, times(3)).create(document.capture());
+ var importedIds = new HashSet<String>();
+ for (var payload : document.getAllValues()) {
+
importedIds.add(JacksonSerializer.getObjectMapper().readTree(payload).get("_id").asText());
+ }
+ assertEquals(Set.of("pipeline:1", "pipeline.2", "pipeline:3.4"),
importedIds);
+ for (var id : importedIds) {
+ verify(resources.managePermissions()).createDefault(id, getClass(),
"importer", false);
+ }
+ }
+
+ @Test
+ void shouldRespectDeselectionForSanitizedArchiveIds() throws IOException {
+ var archive = exportDocumentsWithSpecialIds();
+ var importConfig = new PreviewImportGenerator(null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+ importConfig.getGenericStorageDocuments().forEach(item ->
item.setSelected(false));
+
+ new PerformImportGenerator(importConfig, "importer", null, resources, null)
+ .generate(new ByteArrayInputStream(archive));
+
+ verify(storage, never()).create(anyString());
+ }
+
+ @Test
+ void shouldRejectDocumentIdsThatDoNotMatchArchiveEntries() throws
IOException {
+ var mapper = JacksonSerializer.getObjectMapper();
+ var manifest = new StreamPipesApplicationPackage();
+ manifest.addGenericStorageDocument("different-id");
+ var archive = ZipFileBuilder.create()
+ .addManifest(mapper.writeValueAsString(manifest))
+ .addText("different-id", mapper.writeValueAsString(Map.of("_id",
"pipeline:1", "appDocType", TYPE)))
+ .buildZip();
+ var generator = new PerformImportGenerator(new AssetExportConfiguration(),
"importer", null, resources, null);
+
+ assertThrows(IllegalArgumentException.class, () -> generator.generate(new
ByteArrayInputStream(archive)));
+ verify(storage, never()).create(anyString());
+ }
+
+ private byte[] exportDocumentsWithSpecialIds() throws IOException {
+ var config = new ExportConfiguration();
+ for (var id : List.of("pipeline:1", "pipeline.2", "pipeline:3.4")) {
+ when(storage.findOne(id)).thenReturn(Map.of("_id", id, "appDocType",
TYPE));
+ config.getGenericStorageDocuments().add(new ExportItem(id, id, true));
+ }
+ return export(config);
+ }
+
+ @Test
+ void shouldOnlyPreviewDocumentsLinkedToSelectedAssetsAndTheirChildren()
throws IOException {
+ when(storage.findAll(TYPE)).thenReturn(List.of(first, second,
+ Map.of("_id", "unrelated", "appDocType", TYPE)));
+ when(storage.findOne("asset-1")).thenReturn(Map.of(
+ "assetName", "First asset",
+ "assetLinks", List.of(Map.of("resourceId", "pipeline-1", "queryHint",
"ai-pipeline")),
+ "assets", List.of()));
+ when(storage.findOne("asset-2")).thenReturn(Map.of(
+ "assetName", "Second asset",
+ "assetLinks", List.of(Map.of("resourceId", "pipeline-1", "queryHint",
"ai-pipeline")),
+ "assets", List.of(Map.of("assetLinks", List.of(
+ Map.of("resourceId", "pipeline-2", "queryHint",
"ai-notebook"))))));
+
+ var firstPreview = ExportManager.getExportPreview(List.of("asset-1"),
null, resources, null);
+ assertTrue(firstPreview.getGenericStorageDocuments().isEmpty());
+ assertEquals(List.of("pipeline-1"),
firstPreview.getAssetExportConfiguration().getFirst()
+ .getGenericStorageDocuments().stream()
+ .map(ExportItem::getResourceId).toList());
+
+ var bothPreview = ExportManager.getExportPreview(List.of("asset-1",
"asset-2"), null, resources, null);
+ assertTrue(bothPreview.getGenericStorageDocuments().isEmpty());
+ var firstAsset = bothPreview.getAssetExportConfiguration().getFirst();
+ var secondAsset = bothPreview.getAssetExportConfiguration().get(1);
+ assertEquals("First asset", firstAsset.getAssetName());
+ assertEquals("Second asset", secondAsset.getAssetName());
+ assertEquals(Set.of("pipeline-1", "pipeline-2"), new
HashSet<>(secondAsset.getGenericStorageDocuments().stream()
+ .map(ExportItem::getResourceId).toList()));
+ firstAsset.getGenericStorageDocuments().forEach(item ->
item.setSelected(false));
+
assertTrue(secondAsset.getGenericStorageDocuments().stream().allMatch(ExportItem::isSelected));
+ verify(storage, never()).findAll(TYPE);
+ }
+
+ @Test
+ void shouldNotPreviewRegisteredDocumentsForAssetsWithoutLinks() throws
IOException {
+ when(storage.findOne("asset-1")).thenReturn(Map.of("assetName", "Empty
asset"));
+
+ var config = ExportManager.getExportPreview(List.of("asset-1"), null,
resources, null);
+
+ assertTrue(config.getGenericStorageDocuments().isEmpty());
+ assertTrue(config.getGenericStorageAppDocTypes().stream().noneMatch(item
-> TYPE.equals(item.getResourceId())));
+ }
+
+ @Test
+ void shouldSkipMissingLinksWithoutOfferingBulkFallback() throws IOException {
+ when(storage.findOne("asset-1")).thenReturn(Map.of(
+ "assetName", "Test asset",
+ "assetLinks", List.of(Map.of("resourceId", "missing", "queryHint",
"ai-pipeline"),
+ Map.of("resourceId", "pipeline-1", "queryHint", "ai-pipeline"))));
+ when(storage.findOne("missing")).thenThrow(new IOException("not found"));
+
+ var config = ExportManager.getExportPreview(List.of("asset-1"), null,
resources, null);
+ assertEquals(List.of("pipeline-1"),
config.getAssetExportConfiguration().getFirst()
+
.getGenericStorageDocuments().stream().map(ExportItem::getResourceId).toList());
+ verify(storage, never()).findAll(TYPE);
+
+
+ assertTrue(config.getGenericStorageDocuments().isEmpty());
+ assertTrue(config.getGenericStorageAppDocTypes().stream().noneMatch(item
-> TYPE.equals(item.getResourceId())));
+ }
+
+ private ExportConfiguration preview() {
+ return ExportManager.getExportPreview(List.of(), null, resources, null);
+ }
+
+ private byte[] export(ExportConfiguration config) throws IOException {
+ return ExportManager.getExportPackage(config, null, resources, null);
+ }
+ private static class NamedDocumentResolver extends
GenericStorageDocumentResolver {
+
+ NamedDocumentResolver(IGenericStorage storage) {
+ super(storage);
+ }
+
+ @Override
+ public ExportItem convert(Map<String, Object> document) {
+ var id = document.get("_id").toString();
+ var label = document.get("name");
+ return new ExportItem(id, label instanceof String name &&
!name.isBlank() ? name : id, true);
+ }
+ }
+
+}
diff --git
a/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResolverTest.java
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResolverTest.java
new file mode 100644
index 0000000000..0bf0d124bb
--- /dev/null
+++
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResolverTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.commons.zip.ZipFileExtractor;
+import org.apache.streampipes.export.dataimport.PerformImportGenerator;
+import org.apache.streampipes.export.dataimport.PreviewImportGenerator;
+import org.apache.streampipes.export.resolver.AbstractResolver;
+import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
+import org.apache.streampipes.model.export.AssetExportConfiguration;
+import org.apache.streampipes.model.export.ExportConfiguration;
+import org.apache.streampipes.model.export.ExportItem;
+import org.apache.streampipes.resource.management.SpResourceManager;
+import org.apache.streampipes.serializers.json.JacksonSerializer;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class GenericStorageResolverTest {
+
+ private static final String TYPE = "test:custom-resolver";
+ private static final Function<IGenericStorage, AbstractResolver<Map<String,
Object>>> FACTORY = PipelineDocumentResolver::new;
+ private final IGenericStorage storage = mock(IGenericStorage.class);
+ private final SpResourceManager resources = mock(SpResourceManager.class,
RETURNS_DEEP_STUBS);
+
+ @BeforeEach
+ void setUp() throws IOException {
+ GenericStorageResourceRegistry.INSTANCE.register(TYPE,
+ new ResourceRegistration<>(new PermissionDefaults(getClass(), false),
"Custom pipelines", FACTORY));
+ when(resources.getGenericStorage()).thenReturn(storage);
+ var document = Map.<String, Object>of("_id", "custom-1", "_rev",
"1-revision", "appDocType", TYPE,
+ "title", "Custom name", "running", true, "selectedEndpointUrl",
"http://source";);
+ when(storage.getAllAppDocTypes()).thenReturn(List.of(TYPE));
+ when(storage.findAll(TYPE)).thenReturn(List.of(document));
+ when(storage.findOne("custom-1")).thenAnswer(invocation -> new
HashMap<>(document));
+ }
+
+ @Test
+ void shouldUseCustomConversionInExportAndImportPreviews() throws IOException
{
+ var config = ExportManager.getExportPreview(List.of(), null, resources,
null);
+ assertEquals("Custom name",
config.getGenericStorageDocuments().getFirst().getLabel());
+ assertFalse(config.getGenericStorageDocuments().getFirst().isSelected());
+ config.getGenericStorageDocuments().getFirst().setSelected(true);
+
+ var preview = new PreviewImportGenerator(null, resources, null)
+ .generate(new
ByteArrayInputStream(ExportManager.getExportPackage(config, null, resources,
null)));
+ var item = preview.getGenericStorageDocuments().iterator().next();
+ assertEquals("Custom name", item.getLabel());
+ assertEquals("Custom pipelines", item.getCategory());
+ }
+
+ @Test
+ void shouldSanitizeIndividualAndBulkExports() throws IOException {
+ for (boolean bulk : List.of(false, true)) {
+ var config = new ExportConfiguration();
+ if (bulk) {
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+ } else {
+ config.getGenericStorageDocuments().add(new ExportItem("custom-1",
"Custom name", true));
+ }
+ var archive = ExportManager.getExportPackage(config, null, resources,
null);
+ var files = new ZipFileExtractor(new
ByteArrayInputStream(archive)).extractZipToMap();
+ var exported =
JacksonSerializer.getObjectMapper().readTree(files.get("custom-1"));
+ assertFalse(exported.get("running").asBoolean());
+ assertNull(exported.get("selectedEndpointUrl"));
+ assertNull(exported.get("_rev"));
+ }
+ }
+
+ @Test
+ void shouldUseRegisteredResolverForImportAndOverwrite() throws IOException {
+ var registry = new GenericStorageResourceRegistry();
+ var resolver = spy(new PipelineDocumentResolver(storage));
+ registry.register(TYPE, new ResourceRegistration<>(new
PermissionDefaults(getClass(), false),
+ "Custom pipelines", ignored -> resolver));
+ var config = new ExportConfiguration();
+ config.addGenericStorageAppDocType(new ExportItem(TYPE, TYPE, true));
+ var archive = ExportManager.getExportPackage(config, null, resources,
null);
+ var preview = new PreviewImportGenerator(null, resources,
null).generate(new ByteArrayInputStream(archive));
+ preview.setOverwriteExistingDocuments(true);
+
+ new PerformImportGenerator(preview, "importer", null, resources, null,
registry)
+ .generate(new ByteArrayInputStream(archive));
+
+ verify(resolver).deleteDocument(anyString(), any());
+ verify(resolver).writeDocument(anyString(),
any(AssetExportConfiguration.class));
+ }
+
+ private static class PipelineDocumentResolver extends
GenericStorageDocumentResolver {
+
+ PipelineDocumentResolver(IGenericStorage storage) {
+ super(storage);
+ }
+
+ @Override
+ public Map<String, Object> modifyDocumentForExport(Map<String, Object>
document) {
+ super.modifyDocumentForExport(document);
+ document.put("running", false);
+ document.remove("selectedEndpointUrl");
+ return document;
+ }
+
+ @Override
+ public ExportItem convert(Map<String, Object> document) {
+ return new ExportItem(document.get("_id").toString(),
document.get("title").toString(), true);
+ }
+ }
+}
diff --git
a/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResourceRegistryTest.java
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResourceRegistryTest.java
new file mode 100644
index 0000000000..47c9d5bccb
--- /dev/null
+++
b/streampipes-data-export/src/test/java/org/apache/streampipes/export/GenericStorageResourceRegistryTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.export;
+
+import org.apache.streampipes.commons.exceptions.ElementNotFoundException;
+import org.apache.streampipes.export.resolver.GenericStorageDocumentResolver;
+import org.apache.streampipes.export.resolver.GenericStorageResolverProvider;
+import org.apache.streampipes.storage.api.system.IGenericStorage;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class GenericStorageResourceRegistryTest {
+
+ @Test
+ void shouldRequireResolverFactoriesForResourceRegistrations() {
+ assertThrows(NullPointerException.class,
+ () -> new ResourceRegistration<>(new PermissionDefaults(getClass(),
false), "Resources", null));
+ }
+
+ @Test
+ void shouldRejectConflictingRegistrationsAndAllowRepeatedRegistration() {
+ var registry = new GenericStorageResourceRegistry();
+ var registration = new ResourceRegistration<>(new
PermissionDefaults(getClass(), false),
+ "Resources", GenericStorageDocumentResolver::new);
+ registry.register("test:resource", registration);
+ registry.register("test:resource", registration);
+
+ assertSame(registration, registry.find("test:resource").orElseThrow());
+ assertThrows(IllegalArgumentException.class, () ->
registry.register("test:resource",
+ new ResourceRegistration<>(new PermissionDefaults(getClass(), true),
+ "Resources", GenericStorageDocumentResolver::new)));
+ }
+
+ @Test
+ void shouldExportWithRegisteredSerializationWithoutReloadingDocument()
throws Exception {
+ var storage = mock(IGenericStorage.class);
+ var registry = new GenericStorageResourceRegistry();
+ registry.register("test:resource", new ResourceRegistration<>(new
PermissionDefaults(getClass(), false),
+ "Resources", CustomSerializationResolver::new));
+ when(storage.findOne("resource-1")).thenReturn(Map.of(
+ "_id", "resource-1", "_rev", "1-revision", "appDocType",
"test:resource"));
+
+ var exported = new GenericStorageResolverProvider(storage,
registry).getSerializedDocument("resource-1");
+
+ assertEquals("custom serialization", exported);
+ verify(storage, times(1)).findOne("resource-1");
+ }
+
+ @Test
+ void shouldPreserveGenericFallbackForUnregisteredTypes() throws Exception {
+ var storage = mock(IGenericStorage.class);
+ when(storage.findOne("unknown-1")).thenReturn(Map.of(
+ "_id", "unknown-1", "_rev", "1-revision", "appDocType", "unknown"));
+ var provider = new GenericStorageResolverProvider(storage, new
GenericStorageResourceRegistry());
+
+ assertFalse(provider.getSerializedDocument("unknown-1").contains("_rev"));
+ assertEquals("unknown-1", provider.toExportItem(Map.of("_id", "unknown-1",
"name", "Other"), true).getLabel());
+ }
+
+ @Test
+ void shouldReportMissingDocuments() throws IOException {
+ var storage = mock(IGenericStorage.class);
+ when(storage.findOne("missing")).thenThrow(new IOException("not found"));
+ var provider = new GenericStorageResolverProvider(storage, new
GenericStorageResourceRegistry());
+
+ assertThrows(ElementNotFoundException.class, () ->
provider.getSerializedDocument("missing"));
+ }
+
+ private static class CustomSerializationResolver extends
GenericStorageDocumentResolver {
+
+ CustomSerializationResolver(IGenericStorage storage) {
+ super(storage);
+ }
+
+ @Override
+ public String serializeDocument(Map<String, Object> document) {
+ assertFalse(document.containsKey("_rev"));
+ return "custom serialization";
+ }
+ }
+}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
index f6656a4172..4f6195c3f5 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportConfiguration.java
@@ -29,10 +29,20 @@ public class ExportConfiguration {
private List<AssetExportConfiguration> assetExportConfiguration;
private List<ExportItem> genericStorageAppDocTypes;
+ private List<ExportItem> genericStorageDocuments;
public ExportConfiguration() {
this.assetExportConfiguration = new ArrayList<>();
this.genericStorageAppDocTypes = new ArrayList<>();
+ this.genericStorageDocuments = new ArrayList<>();
+ }
+
+ public List<ExportItem> getGenericStorageDocuments() {
+ return genericStorageDocuments;
+ }
+
+ public void setGenericStorageDocuments(List<ExportItem>
genericStorageDocuments) {
+ this.genericStorageDocuments = genericStorageDocuments;
}
public List<AssetExportConfiguration> getAssetExportConfiguration() {
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
index cb62711a05..79ed281dc5 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/export/ExportItem.java
@@ -23,6 +23,7 @@ public class ExportItem {
private String resourceId;
private String label;
private boolean selected;
+ private String category;
public ExportItem() {
}
@@ -33,6 +34,14 @@ public class ExportItem {
this.selected = selected;
}
+ public String getCategory() {
+ return category;
+ }
+
+ public void setCategory(String category) {
+ this.category = category;
+ }
+
public String getResourceId() {
return resourceId;
}
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index b4144727fd..92cf0d4c2e 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -17,7 +17,7 @@
* under the License.
*/
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2026-10-13
16:58:25.
+// Generated using typescript-generator version 3.2.1263 on 2026-10-26
15:02:34.
export class NamedStreamPipesEntity implements Storable {
'@class':
@@ -1993,6 +1993,7 @@ export class ExportConfig {
export class ExportConfiguration {
assetExportConfiguration: AssetExportConfiguration[];
genericStorageAppDocTypes: ExportItem[];
+ genericStorageDocuments: ExportItem[];
static fromData(
data: ExportConfiguration,
@@ -2008,11 +2009,15 @@ export class ExportConfiguration {
instance.genericStorageAppDocTypes = __getCopyArrayFn(
ExportItem.fromData,
)(data.genericStorageAppDocTypes);
+ instance.genericStorageDocuments = __getCopyArrayFn(
+ ExportItem.fromData,
+ )(data.genericStorageDocuments);
return instance;
}
}
export class ExportItem {
+ category: string;
label: string;
resourceId: string;
selected: boolean;
@@ -2022,6 +2027,7 @@ export class ExportItem {
return data;
}
const instance = target || new ExportItem();
+ instance.category = data.category;
instance.label = data.label;
instance.resourceId = data.resourceId;
instance.selected = data.selected;
diff --git a/ui/src/app/assets/dialog/move-asset-dialog.component.ts
b/ui/src/app/assets/dialog/move-asset-dialog.component.ts
index 16109c5d0c..d3e8cf8886 100644
--- a/ui/src/app/assets/dialog/move-asset-dialog.component.ts
+++ b/ui/src/app/assets/dialog/move-asset-dialog.component.ts
@@ -19,7 +19,6 @@
import { NestedTreeControl } from '@angular/cdk/tree';
import { Component, inject, Input } from '@angular/core';
import {
- FlexDirective,
LayoutAlignDirective,
LayoutDirective,
LayoutGapDirective,
@@ -72,7 +71,6 @@ export type MoveAssetDialogResult =
MatButton,
MatIconButton,
MatIcon,
- FlexDirective,
LayoutDirective,
LayoutAlignDirective,
LayoutGapDirective,
diff --git a/ui/src/app/configuration/export/data-export-import.component.ts
b/ui/src/app/configuration/export/data-export-import.component.ts
index ddaaae54a4..6f6d41aee5 100644
--- a/ui/src/app/configuration/export/data-export-import.component.ts
+++ b/ui/src/app/configuration/export/data-export-import.component.ts
@@ -211,6 +211,7 @@ export class SpDataExportImportComponent implements OnInit {
resourceId: label._id!,
label: label.label,
selected: true,
+ category: null,
}));
referencedSites[asset.elementId] = [...siteIds]
.map(siteId => sitesById.get(siteId))
@@ -219,6 +220,7 @@ export class SpDataExportImportComponent implements OnInit {
resourceId: site._id,
label: site.label,
selected: true,
+ category: null,
}));
});
diff --git
a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.spec.ts
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.spec.ts
new file mode 100644
index 0000000000..db89fd414e
--- /dev/null
+++
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.spec.ts
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { TestBed } from '@angular/core/testing';
+import { DialogRef } from '@streampipes/shared-ui';
+import {
+ AssetExportConfiguration,
+ ExportConfiguration,
+} from '@streampipes/platform-services';
+import { of } from 'rxjs';
+import { describe, expect, it } from 'vitest';
+import { DataExportService } from '../data-export.service';
+import { SpDataExportDialogComponent } from './data-export-dialog.component';
+
+function asset(assetId: string, assetName: string): AssetExportConfiguration {
+ return Object.assign(new AssetExportConfiguration(), {
+ assetId,
+ assetName,
+ adapters: [],
+ dashboards: [],
+ dataViews: [],
+ dataSources: [],
+ dataLakeMeasures: [],
+ files: [],
+ pipelines: [],
+ labels: [],
+ sites: [],
+ genericStorageDocuments: [],
+ });
+}
+
+function component(preview: ExportConfiguration): SpDataExportDialogComponent {
+ TestBed.configureTestingModule({
+ providers: [
+ { provide: DialogRef, useValue: {} },
+ {
+ provide: DataExportService,
+ useValue: { getExportPreview: () => of(preview) },
+ },
+ ],
+ });
+ return TestBed.runInInjectionContext(
+ () => new SpDataExportDialogComponent(),
+ );
+}
+
+function preview(assets: AssetExportConfiguration[]): ExportConfiguration {
+ return Object.assign(new ExportConfiguration(), {
+ assetExportConfiguration: assets,
+ genericStorageAppDocTypes: [],
+ genericStorageDocuments: [
+ {
+ resourceId: 'shared',
+ label: 'Shared pipeline',
+ category: 'AI Pipelines',
+ selected: false,
+ },
+ {
+ resourceId: 'unrelated',
+ label: 'Other pipeline',
+ category: 'AI Pipelines',
+ selected: false,
+ },
+ ],
+ });
+}
+
+describe('generic export asset rows', () => {
+ it('shows named asset rows with independent selections for a shared
resource', () => {
+ const config = preview([asset('a', 'Asset A'), asset('b', 'Asset B')]);
+ const dialog = component(config);
+ dialog.selectedAssets = ['a', 'b'];
+ for (const assetConfig of config.assetExportConfiguration) {
+ assetConfig.genericStorageDocuments = [
+ { ...config.genericStorageDocuments[0] },
+ ];
+ }
+ config.genericStorageDocuments = [];
+ dialog.ngOnInit();
+
+ const rows = dialog.getAssetsForSection('genericStorage::AI
Pipelines');
+ expect(rows.map(row => row.assetName)).toEqual(['Asset A', 'Asset B']);
+ expect(rows.map(row => row.items.map(item =>
item.resourceId))).toEqual(
+ [['shared'], ['shared']],
+ );
+ expect(config.genericStorageDocuments).toEqual([]);
+ dialog.updateSelectedItems('genericStorage::AI Pipelines', rows[0],
[]);
+ expect(
+ config.assetExportConfiguration[0].genericStorageDocuments[0]
+ .selected,
+ ).toBe(false);
+ expect(
+ config.assetExportConfiguration[1].genericStorageDocuments[0]
+ .selected,
+ ).toBe(true);
+ });
+
+ it('retains the global category when no assets are selected', () => {
+ const config = preview([]);
+ const dialog = component(config);
+ dialog.selectedAssets = [];
+ dialog.ngOnInit();
+ expect(
+ dialog.getAssetsForSection('genericStorage::AI
Pipelines')[0].items,
+ ).toHaveLength(2);
+ expect(config.genericStorageDocuments).toHaveLength(2);
+ });
+});
diff --git
a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
index 19479290e5..2f03df3b92 100644
---
a/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
+++
b/ui/src/app/configuration/export/export-dialog/data-export-dialog.component.ts
@@ -60,7 +60,7 @@ interface SectionAssetConfiguration {
items: ExportItem[];
}
-type ExportSectionKey =
+type AssetExportSectionKey =
| 'adapters'
| 'dashboards'
| 'dataViews'
@@ -109,7 +109,7 @@ export class SpDataExportDialogComponent implements OnInit {
@Input()
referencedSites: Record<string, ExportItem[]> = {};
- exportSections = [
+ private readonly resourceSections = [
{ key: 'adapters', title: 'Adapters' },
{ key: 'dashboards', title: 'Dashboards' },
{ key: 'dataViews', title: 'Charts' },
@@ -121,6 +121,10 @@ export class SpDataExportDialogComponent implements OnInit
{
{ key: 'sites', title: 'Sites' },
] as const;
+ exportSections: { key: string; title: string }[] = [
+ ...this.resourceSections,
+ ];
+
preview: ExportConfiguration;
exportInProgress = false;
exportAllSelections: Record<string, boolean> = {};
@@ -141,12 +145,12 @@ export class SpDataExportDialogComponent implements
OnInit {
getExportItems(
config: AssetExportConfiguration,
- key: (typeof this.exportSections)[number]['key'],
+ key: AssetExportSectionKey,
): ExportItem[] {
return config[key];
}
- getAssetsForSection(key: ExportSectionKey): SectionAssetConfiguration[] {
+ getAssetsForSection(key: string): SectionAssetConfiguration[] {
return this.sectionAssets[key] ?? [];
}
@@ -165,7 +169,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
selectAllItemsForAsset(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
select: boolean,
): void {
@@ -177,7 +181,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
isExportAllSelected(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
): boolean {
return this.exportAllSelections[
@@ -186,7 +190,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
changeExportAllSelection(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
event: MatSlideToggleChange,
): void {
@@ -203,7 +207,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
getSelectedItemResourceIds(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
): string[] {
return (
@@ -214,7 +218,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
updateSelectedItems(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
selectedResourceIds: string[],
): void {
@@ -276,7 +280,8 @@ export class SpDataExportDialogComponent implements OnInit {
private initializeSectionAssets(): void {
this.sectionAssets = {};
- this.exportSections.forEach(section => {
+ this.exportSections = [...this.resourceSections];
+ this.resourceSections.forEach(section => {
this.sectionAssets[section.key] =
this.preview.assetExportConfiguration
.map(config => ({
@@ -286,11 +291,44 @@ export class SpDataExportDialogComponent implements
OnInit {
}))
.filter(config => config.items.length > 0);
});
+
+ const genericSections = new Map<string, SectionAssetConfiguration[]>();
+ for (const config of this.preview.assetExportConfiguration) {
+ for (const [title, items] of Object.entries(
+ groupExportItems(config.genericStorageDocuments),
+ )) {
+ const rows = genericSections.get(title) ?? [];
+ rows.push({
+ assetId: config.assetId,
+ assetName: config.assetName,
+ items,
+ });
+ genericSections.set(title, rows);
+ }
+ }
+ for (const [title, items] of Object.entries(
+ groupExportItems(this.preview.genericStorageDocuments),
+ )) {
+ const rows = genericSections.get(title) ?? [];
+ rows.push({
+ assetId: `genericStorage::${title}`,
+ assetName: '',
+ items,
+ });
+ genericSections.set(title, rows);
+ }
+ for (const [title, rows] of [...genericSections].sort(
+ ([left], [right]) => left.localeCompare(right),
+ )) {
+ const key = `genericStorage::${title}`;
+ this.exportSections.push({ key, title });
+ this.sectionAssets[key] = rows;
+ }
}
private sortPreviewItems(): void {
this.preview.assetExportConfiguration.forEach(config => {
- this.exportSections.forEach(section => {
+ this.resourceSections.forEach(section => {
this.sortExportItems(config[section.key]);
});
});
@@ -303,7 +341,7 @@ export class SpDataExportDialogComponent implements OnInit {
}
private getExportAllSelectionKey(
- key: ExportSectionKey,
+ key: string,
assetConfig: SectionAssetConfiguration,
): string {
return `${key}::${assetConfig.assetId}`;
@@ -336,3 +374,19 @@ export class SpDataExportDialogComponent implements OnInit
{
return [...existingItems, ...missingItems];
}
}
+
+export function groupExportItems(
+ items: ExportItem[] = [],
+): Record<string, ExportItem[]> {
+ return [...(items ?? [])]
+ .sort((left, right) => left.label.localeCompare(right.label))
+ .reduce(
+ (groups, item) => {
+ if (item.category) {
+ (groups[item.category] ??= []).push(item);
+ }
+ return groups;
+ },
+ Object.create(null) as Record<string, ExportItem[]>,
+ );
+}
diff --git
a/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
index 53da522613..633a1444a6 100644
---
a/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
+++
b/ui/src/app/configuration/export/export-dialog/generic-storage-items/generic-storage-items.component.ts
@@ -63,6 +63,7 @@ export class GenericStorageItemsComponent {
resourceId: this.newDocumentId,
label: this.newDocumentId,
selected: true,
+ category: null,
});
}
diff --git
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
index ffb54ca76c..430e19be35 100644
---
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
+++
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.html
@@ -148,11 +148,23 @@
[exportItems]="importConfiguration.sites"
[sectionTitle]="'Sites' | translate"
></sp-data-export-item>
- <sp-generic-storage-items
- [exportItems]="importConfiguration.genericStorageDocuments"
- [importMode]="true"
- >
- </sp-generic-storage-items>
+ @for (
+ category of groupExportItems(
+ importConfiguration.genericStorageDocuments
+ ) | keyvalue;
+ track category.key
+ ) {
+ <sp-data-export-item
+ [exportItems]="category.value"
+ [sectionTitle]="category.key | translate"
+ ></sp-data-export-item>
+ }
+ @if (genericDocuments.length > 0) {
+ <sp-generic-storage-items
+ [exportItems]="genericDocuments"
+ [importMode]="true"
+ ></sp-generic-storage-items>
+ }
@if (currentImportStep === 1) {
<sp-split-section
[level]="3"
diff --git
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
index 71eb45abd6..03908451ff 100644
---
a/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
+++
b/ui/src/app/configuration/export/import-dialog/data-import-dialog.component.ts
@@ -51,6 +51,8 @@ import { GenericStorageItemsComponent } from
'../export-dialog/generic-storage-i
import { MatCheckbox } from '@angular/material/checkbox';
import { FormsModule } from '@angular/forms';
import { MatDivider } from '@angular/material/divider';
+import { KeyValuePipe } from '@angular/common';
+import { groupExportItems } from
'../export-dialog/data-export-dialog.component';
@Component({
selector: 'sp-data-import-dialog',
@@ -78,6 +80,7 @@ import { MatDivider } from '@angular/material/divider';
SpSpinnerComponent,
MatDivider,
TranslatePipe,
+ KeyValuePipe,
],
})
export class SpDataImportDialogComponent {
@@ -88,6 +91,13 @@ export class SpDataImportDialogComponent {
private translateService = inject(TranslateService);
currentImportStep = 0;
+ readonly groupExportItems = groupExportItems;
+
+ get genericDocuments(): ExportItem[] {
+ return this.importConfiguration.genericStorageDocuments.filter(
+ item => !item.category,
+ );
+ }
inputValue: string;
fileName: string;
