This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 8e51d30779 Data Retention Export to s3 based object storage (#3824)
8e51d30779 is described below
commit 8e51d30779d844a2ccede0ad32da5966022188a1
Author: Jacqueline Höllig <[email protected]>
AuthorDate: Wed Oct 15 08:47:38 2025 +0200
Data Retention Export to s3 based object storage (#3824)
---
.../apache/streampipes/commons/constants/Envs.java | 2 +
.../commons/environment/DefaultEnvironment.java | 4 +
.../commons/environment/Environment.java | 1 +
streampipes-data-explorer-export/pom.xml | 31 ++++
.../export/ObjectStorge/ExportProviderFactory.java | 19 +-
.../export/ObjectStorge/LocalFolder.java | 7 +-
.../dataexplorer/export/ObjectStorge/S3.java | 86 +++++++++
.../DefaultExportProviderConfig.java} | 17 +-
.../configuration/DefaultSpCoreConfiguration.java | 1 +
.../configuration/ExportProviderSettings.java | 115 ++++++++++++
.../ProviderType.java} | 10 +-
.../model/configuration/SpCoreConfiguration.java | 12 ++
.../model/datalake/DataLakeMeasure.java | 4 +
.../model/datalake/RetentionExportConfig.java | 28 ++-
.../model/datalake/RetentionTimeConfig.java | 35 +++-
.../admin/ExportProviderConfigurationResource.java | 105 +++++++++++
.../rest/impl/datalake/DataLakeResource.java | 15 ++
streampipes-service-core/pom.xml | 1 +
.../service/core/scheduler/DataLakeScheduler.java | 106 ++++++++---
ui/deployment/i18n/de.json | 88 ++++++---
ui/deployment/i18n/en.json | 94 +++++++---
.../src/lib/apis/datalake-rest.service.ts | 6 +-
.../src/lib/apis/export-provider.service.ts | 66 +++++++
.../src/lib/model/gen/streampipes-model-client.ts | 5 +-
.../src/lib/model/gen/streampipes-model.ts | 35 ++--
ui/src/app/configuration/configuration.module.ts | 4 +
.../datalake-configuration.component.html | 197 +++++++++++++++++++++
.../datalake-configuration.component.ts | 80 +++++++--
.../select-export/select-format.component.html | 91 ++++++++--
.../select-export/select-format.component.scss | 19 ++
.../select-export/select-format.component.ts | 93 +++++++++-
.../data-retention-dialog.component.html | 10 ++
.../data-retention-dialog.component.ts | 54 ++++--
.../delete-export-provider-dialog.component.html | 82 +++++++++
.../delete-export-provider-dialog.component.ts | 51 ++++++
.../export-provider-dialog.component.html | 121 +++++++++++++
.../export-provider-dialog.component.ts | 133 ++++++++++++++
37 files changed, 1681 insertions(+), 147 deletions(-)
diff --git
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
index bd06f7317a..49622cf9d1 100644
---
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
+++
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/constants/Envs.java
@@ -142,6 +142,8 @@ public enum Envs {
// Retention Local File
SP_RETENTION_LOCAL_DIR("SP_RETENTION_LOCAL_DIR", "./ArchivedData"),
+ SP_DATALAKE_SCHEDULER_CRON("SP_DATALAKE_SCHEDULER_CRON", "0 1 0 * * 6"),//
CronJob Scheduled every Saturday (6) 00:01 //@Scheduled(cron = "0 */2 * * *
+ // *") //Cron Job in Dev Setting; Running
every 2 min
// Logging
SP_LOGGING_FILE_ENABLED("SP_LOGGING_FILE_ENABLED", "false"),
diff --git
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
index c8f5f0686c..fc3e765b51 100644
---
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
+++
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/DefaultEnvironment.java
@@ -442,4 +442,8 @@ public class DefaultEnvironment implements Environment {
public StringEnvironmentVariable getRetentionLocalDir() {
return new StringEnvironmentVariable(Envs.SP_RETENTION_LOCAL_DIR);
}
+ @Override
+ public StringEnvironmentVariable getDatalakeSchedulerCron() {
+ return new StringEnvironmentVariable(Envs.SP_DATALAKE_SCHEDULER_CRON);
+ }
}
diff --git
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
index 07ed38c9df..a670ed71a2 100644
---
a/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
+++
b/streampipes-commons/src/main/java/org/apache/streampipes/commons/environment/Environment.java
@@ -168,4 +168,5 @@ public interface Environment {
StringEnvironmentVariable getFileLoggingPattern();
StringEnvironmentVariable getRetentionLocalDir();
+ StringEnvironmentVariable getDatalakeSchedulerCron();
}
diff --git a/streampipes-data-explorer-export/pom.xml
b/streampipes-data-explorer-export/pom.xml
index b0a7452c6a..303d34ac8c 100644
--- a/streampipes-data-explorer-export/pom.xml
+++ b/streampipes-data-explorer-export/pom.xml
@@ -78,6 +78,37 @@
<artifactId>spring-webmvc</artifactId>
<version>5.3.32</version>
</dependency>
+ <dependency>
+ <groupId>software.amazon.awssdk</groupId>
+ <artifactId>s3</artifactId>
+ <version>2.25.14</version> <!-- Use latest, 2.25.14-->
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-unix-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-resolver</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-classes-epoll</artifactId> <!--
Exclude this one -->
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Test dependencies -->
<dependency>
diff --git
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/ExportProviderFactory.java
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/ExportProviderFactory.java
index c4c515475d..bd0b64823d 100644
---
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/ExportProviderFactory.java
+++
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/ExportProviderFactory.java
@@ -17,14 +17,25 @@
*/
package org.apache.streampipes.dataexplorer.export.ObjectStorge;
-import org.apache.streampipes.model.datalake.ExportProviderSettings;
+import org.apache.streampipes.model.configuration.ExportProviderSettings;
+import org.apache.streampipes.model.configuration.ProviderType;
public class ExportProviderFactory {
- public static IObjectStorage createExportProvider(String providerType,
String measurementName, ExportProviderSettings settings, String format) throws
Exception {
+
+
+ public static IObjectStorage createExportProvider(
+ ProviderType providerType,
+ String measurementName,
+ ExportProviderSettings settings,
+ String format) throws Exception {
+
switch (providerType) {
- case "local":
+ case FOLDER:
return new LocalFolder(measurementName, format);
- //Additional Providers can be added here
+
+ case S3:
+ return new S3(measurementName, format, settings);
+
default:
throw new IllegalArgumentException("Unsupported provider: " +
providerType);
}
diff --git
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/LocalFolder.java
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/LocalFolder.java
index eeb1e6ffec..1e59f24d9c 100644
---
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/LocalFolder.java
+++
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/LocalFolder.java
@@ -17,6 +17,7 @@
*/
package org.apache.streampipes.dataexplorer.export.ObjectStorge;
+import org.apache.streampipes.commons.environment.Environments;
import
org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
import java.io.FileOutputStream;
@@ -31,9 +32,11 @@ public class LocalFolder implements IObjectStorage {
private final Path filePath;
- public LocalFolder(String measurementName, String format) throws Exception
{
+ public LocalFolder(String measurementName, String format) throws
RuntimeException, IOException {
-
Files.createDirectories(Paths.get(System.getenv("SP_RETENTION_LOCAL_DIR") + "/"
+ measurementName));
+ var env = Environments.getEnvironment();
+
+
Files.createDirectories(Paths.get(env.getRetentionLocalDir().getValueOrDefault()
+ "/" + measurementName));
this.filePath = Paths.get(System.getenv("SP_RETENTION_LOCAL_DIR") +
"/" + measurementName + "/dump_"
+ Instant.now().toString() + "." + format);
diff --git
a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/S3.java
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/S3.java
new file mode 100644
index 0000000000..885e4f9e90
--- /dev/null
+++
b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ObjectStorge/S3.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.dataexplorer.export.ObjectStorge;
+
+
+import org.apache.streampipes.model.configuration.ExportProviderSettings;
+import
org.apache.streampipes.user.management.encryption.SecretEncryptionManager;
+
+import
org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.time.Instant;
+
+
+public class S3 implements IObjectStorage{
+
+ private final String fileName;
+ private final S3Client s3;
+ private final String bucketName;
+
+ public S3(String measurementName, String format, ExportProviderSettings
settings) throws RuntimeException {
+
+ this.s3 = S3Client.builder()
+ .endpointOverride(URI.create(settings.getEndPoint()))
+ .region(Region.of(settings.getAwsRegion()))
+ .credentialsProvider(
+ StaticCredentialsProvider.create(
+
AwsBasicCredentials.create(settings.getAccessKey(),
SecretEncryptionManager.decrypt(settings.getSecretKey()))
+ )
+ )
+ .build();
+
+ this.bucketName = settings.getBucketName();
+ this.fileName = "/" + measurementName + "/dump_"
+ + Instant.now().toString() + "." + format;
+
+ }
+
+ @Override
+ public void store(StreamingResponseBody datastream) throws IOException {
+
+ PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+ .bucket(bucketName)
+ .key(fileName)
+ .build();
+
+ ByteArrayOutputStream byteArrayOutputStream = new
ByteArrayOutputStream();
+
+
+ datastream.writeTo(byteArrayOutputStream);
+
+
+ byte[] data = byteArrayOutputStream.toByteArray();
+
+ RequestBody requestBody = RequestBody.fromBytes(data);
+
+ // Upload to S3
+ this.s3.putObject(putObjectRequest, requestBody);
+ this.s3.close();
+ }
+
+
+}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultExportProviderConfig.java
similarity index 70%
copy from
streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
copy to
streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultExportProviderConfig.java
index 43e1b2199f..de17db5e51 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultExportProviderConfig.java
@@ -16,10 +16,17 @@
*
*/
-package org.apache.streampipes.model.datalake;
+package org.apache.streampipes.model.configuration;
-public record RetentionTimeConfig(
- DataRetentionConfig dataRetentionConfig,
- RetentionExportConfig exportConfig
- ) {}
+import java.util.List;
+public class DefaultExportProviderConfig {
+
+public List<ExportProviderSettings> make() {
+
+ var defaultSettings = new ExportProviderSettings(
+ ProviderType.FOLDER, "FOLDER", "", "", "", "", "US_EAST_1");
+
+ return List.of(defaultSettings);
+ }
+}
\ No newline at end of file
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java
index 0514006380..6522d7358c 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/DefaultSpCoreConfiguration.java
@@ -36,6 +36,7 @@ public class DefaultSpCoreConfiguration {
coreCfg.setFilesDir(makeFileLocation());
coreCfg.setAssetDir(makeAssetLocation());
coreCfg.setLocalAuthConfig(LocalAuthConfig.fromDefaults(getJwtSecret()));
+ coreCfg.setExportProviderSettings(new
DefaultExportProviderConfig().make());
return coreCfg;
}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ExportProviderSettings.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ExportProviderSettings.java
new file mode 100644
index 0000000000..5feaeabb31
--- /dev/null
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ExportProviderSettings.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.model.configuration;
+
+import org.apache.streampipes.model.shared.annotation.TsModel;
+
+@TsModel
+public class ExportProviderSettings {
+
+ private boolean secretEncrypted;
+
+ private ProviderType providerType;
+ private String providerId;
+ private String accessKey;
+ private String secretKey;
+ private String bucketName;
+ private String endPoint;
+ private String awsRegion;
+
+ public ExportProviderSettings(ProviderType providerType, String
providerId, String accessKey, String secretKey, String bucketName, String
endPoint, String awsRegion) {
+ this.providerType = providerType;
+ this.providerId = providerId;
+ this.accessKey = accessKey;
+ this.secretKey = secretKey;
+ this.bucketName = bucketName;
+ this.endPoint = endPoint;
+ this.awsRegion = awsRegion;
+ }
+
+ public ProviderType getProviderType() {
+ return providerType;
+ }
+ public String getAwsRegion() {
+ return awsRegion;
+ }
+
+ public void setAwsRegion(String awsRegion) {
+ this.awsRegion = awsRegion;
+ }
+
+ public void setProviderType(ProviderType providerType) {
+ this.providerType = providerType;
+ }
+ public String getAccessKey() {
+ return accessKey;
+ }
+ public void setAccessKey(String accessKey) {
+ this.accessKey = accessKey;
+ }
+ public String getSecretKey() {
+ return secretKey;
+ }
+ public void setSecretKey(String secretKey) {
+ this.secretKey = secretKey;
+ }
+ public String getBucketName() {
+ return bucketName;
+ }
+ public void setBucketName(String bucketName) {
+ this.bucketName = bucketName;
+ }
+
+ public String getEndPoint() {
+ return endPoint;
+ }
+
+ public void setEndPoint(String endPoint) {
+ this.endPoint = endPoint;
+ }
+
+ public String getProviderId() {
+ return providerId;
+ }
+
+ public void setProviderId(String providerId) {
+ this.providerId = providerId;
+ }
+
+ public static ExportProviderSettings fromDefaults() {
+ ExportProviderSettings config = new ExportProviderSettings();
+ config.setSecretEncrypted(false);
+
+ return config;
+ }
+
+ public ExportProviderSettings() {
+
+ }
+
+ public boolean isSecretEncrypted() {
+ return secretEncrypted;
+ }
+
+ public void setSecretEncrypted(boolean secretEncrypted) {
+ this.secretEncrypted = secretEncrypted;
+ }
+}
+
+
+
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/ExportProviderSettings.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ProviderType.java
similarity index 87%
rename from
streampipes-model/src/main/java/org/apache/streampipes/model/datalake/ExportProviderSettings.java
rename to
streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ProviderType.java
index fab05be587..d46d648152 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/ExportProviderSettings.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/ProviderType.java
@@ -15,8 +15,10 @@
* limitations under the License.
*
*/
-package org.apache.streampipes.model.datalake;
-public record ExportProviderSettings(
- String providerType
-) {}
\ No newline at end of file
+package org.apache.streampipes.model.configuration;
+
+public enum ProviderType {
+ FOLDER,
+ S3;
+}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java
index b5766bd3f6..788ecbf1e7 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/configuration/SpCoreConfiguration.java
@@ -20,6 +20,8 @@ package org.apache.streampipes.model.configuration;
import com.google.gson.annotations.SerializedName;
+import java.util.List;
+
public class SpCoreConfiguration {
public static final String ID = "core";
@@ -40,6 +42,8 @@ public class SpCoreConfiguration {
private String assetDir;
private String filesDir;
+ private List<ExportProviderSettings> exportProviderSettings;
+
public SpCoreConfiguration() {
this.locationConfig = new LocationConfig(false, "", "");
}
@@ -139,4 +143,12 @@ public class SpCoreConfiguration {
public void setLocationConfig(LocationConfig locationConfig) {
this.locationConfig = locationConfig;
}
+
+ public List<ExportProviderSettings> getExportProviderSettings() {
+ return exportProviderSettings;
+ }
+
+ public void setExportProviderSettings(List<ExportProviderSettings>
exportProviderSettings) {
+ this.exportProviderSettings = exportProviderSettings;
+ }
}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
index 50eefd8da3..ae832d38ea 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/DataLakeMeasure.java
@@ -146,6 +146,10 @@ public class DataLakeMeasure implements Storable {
this.retentionTime = retentionTime;
}
+ public void deleteRetentionTime() {
+ this.retentionTime = null;
+ }
+
public RetentionTimeConfig getRetentionTime() {
return retentionTime;
}
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionExportConfig.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionExportConfig.java
index d1fbf544e3..f126a63744 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionExportConfig.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionExportConfig.java
@@ -17,8 +17,28 @@
*/
package org.apache.streampipes.model.datalake;
+public class RetentionExportConfig {
+
+ private ExportConfig exportConfig;
+ private String exportProviderId;
-public record RetentionExportConfig(
- ExportConfig exportConfig,
- ExportProviderSettings exportProviderSettings
-) {}
+ public RetentionExportConfig(ExportConfig exportConfig, String
exportProviderId) {
+ this.exportConfig = exportConfig;
+ this.exportProviderId = exportProviderId;
+ }
+
+ public ExportConfig getExportConfig() {
+ return exportConfig;
+ }
+
+ public void setExportConfig(ExportConfig exportConfig) {
+ this.exportConfig = exportConfig;
+ }
+ public String getExportProviderId() {
+ return exportProviderId;
+ }
+
+ public void setExportProviderId(String exportProviderId) {
+ this.exportProviderId = exportProviderId;
+ }
+}
\ No newline at end of file
diff --git
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
index 43e1b2199f..1afb3327c2 100644
---
a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
+++
b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/RetentionTimeConfig.java
@@ -18,8 +18,35 @@
package org.apache.streampipes.model.datalake;
-public record RetentionTimeConfig(
- DataRetentionConfig dataRetentionConfig,
- RetentionExportConfig exportConfig
- ) {}
+public class RetentionTimeConfig {
+
+ private DataRetentionConfig dataRetentionConfig;
+ private RetentionExportConfig retentionExportConfig;
+
+ public RetentionTimeConfig() {}
+
+
+ public RetentionTimeConfig(
+ DataRetentionConfig dataRetentionConfig,
+ RetentionExportConfig retentionExportConfig) {
+ this.dataRetentionConfig = dataRetentionConfig;
+ this.retentionExportConfig = retentionExportConfig;
+ }
+
+ public DataRetentionConfig getDataRetentionConfig() {
+ return dataRetentionConfig;
+ }
+
+ public void setDataRetentionConfig(DataRetentionConfig
dataRetentionConfig) {
+ this.dataRetentionConfig = dataRetentionConfig;
+ }
+
+ public RetentionExportConfig getRetentionExportConfig() {
+ return retentionExportConfig;
+ }
+
+ public void setRetentionExportConfig(RetentionExportConfig exportConfig) {
+ this.retentionExportConfig = exportConfig;
+ }
+}
\ No newline at end of file
diff --git
a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExportProviderConfigurationResource.java
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExportProviderConfigurationResource.java
new file mode 100644
index 0000000000..774e9805be
--- /dev/null
+++
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/admin/ExportProviderConfigurationResource.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.streampipes.rest.impl.admin;
+
+import org.apache.streampipes.model.configuration.ExportProviderSettings;
+import
org.apache.streampipes.rest.core.base.impl.AbstractAuthGuardedRestResource;
+import org.apache.streampipes.rest.security.AuthConstants;
+import
org.apache.streampipes.user.management.encryption.SecretEncryptionManager;
+
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PutMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+@RestController
+@RequestMapping("/api/v2/admin/exportprovider-config")
+public class ExportProviderConfigurationResource extends
AbstractAuthGuardedRestResource {
+
+ @GetMapping(produces = MediaType.APPLICATION_JSON_VALUE)
+ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
+ public ResponseEntity<List<ExportProviderSettings>>
getExportProviderConfiguration() {
+ return
ok(getSpCoreConfigurationStorage().get().getExportProviderSettings());
+ }
+
+ @GetMapping(value = "/{providerId}", produces =
MediaType.APPLICATION_JSON_VALUE)
+@PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
+public ResponseEntity<ExportProviderSettings>
getExportProviderSettingById(@PathVariable String providerId) {
+ return
getSpCoreConfigurationStorage().get().getExportProviderSettings().stream()
+ .filter(setting ->
setting.getProviderId().equalsIgnoreCase(providerId))
+ .findFirst()
+ .map(ResponseEntity::ok)
+ .orElse(ResponseEntity.notFound().build());
+}
+
+ @PutMapping(consumes = MediaType.APPLICATION_JSON_VALUE)
+ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
+ public ResponseEntity<Void> updateExportProviderConfiguration(@RequestBody
ExportProviderSettings config) {
+
+ if (!config.isSecretEncrypted()) {
+
config.setSecretKey(SecretEncryptionManager.encrypt(config.getSecretKey()));
+ config.setSecretEncrypted(true);
+ }
+ var storage = getSpCoreConfigurationStorage();
+ var cfg = storage.get();
+
+ List<ExportProviderSettings> providerSettings =
cfg.getExportProviderSettings();
+ if (providerSettings == null) {
+ providerSettings = new ArrayList<>();
+ }
+
+ List<ExportProviderSettings> providerSettingsWithoutExisting =
providerSettings.stream()
+ .filter(existing -> existing != null &&
!existing.getProviderId().equals(config.getProviderId()))
+ .collect(Collectors.toList());
+
+ providerSettingsWithoutExisting.add(config);
+
+ cfg.setExportProviderSettings(providerSettingsWithoutExisting);
+ storage.updateElement(cfg);
+
+ return ok();
+ }
+
+ @DeleteMapping(value = "/{providerId}", produces =
MediaType.APPLICATION_JSON_VALUE)
+ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
+ public ResponseEntity<Void> deleteExportProviderConfiguration(@PathVariable
String providerId) {
+
+ List<ExportProviderSettings> allProviders =
getSpCoreConfigurationStorage().get().getExportProviderSettings();
+
+ List<ExportProviderSettings> filteredProviders = allProviders.stream()
+ .filter(provider -> !provider.getProviderId().equals(providerId))
+ .collect(Collectors.toList());
+
+ var storage = getSpCoreConfigurationStorage();
+ var cfg = storage.get();
+ cfg.setExportProviderSettings(filteredProviders);
+ storage.updateElement(cfg);
+ return ok();
+ }
+
+}
diff --git
a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
index 66ffa2a82d..2b93b2476c 100644
---
a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
+++
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
@@ -31,6 +31,7 @@ import
org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
import org.apache.streampipes.model.message.Notifications;
import org.apache.streampipes.model.monitoring.SpLogMessage;
import org.apache.streampipes.rest.core.base.impl.AbstractRestResource;
+import org.apache.streampipes.rest.security.AuthConstants;
import org.apache.streampipes.rest.shared.exception.SpMessageException;
import io.swagger.v3.oas.annotations.Operation;
@@ -46,6 +47,7 @@ import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@@ -405,6 +407,7 @@ public class DataLakeResource extends AbstractRestResource {
path = "/{elementId}/cleanup",
produces = MediaType.APPLICATION_JSON_VALUE,
consumes = MediaType.APPLICATION_JSON_VALUE)
+ @PreAuthorize(AuthConstants.IS_ADMIN_ROLE)
@Operation(summary = "Sets the retention mechanism for a certain
measurement", tags = {"Data Lake"},
responses = {
@ApiResponse(
@@ -427,6 +430,18 @@ public class DataLakeResource extends AbstractRestResource
{
return ok();
}
+@DeleteMapping(path = "/{elementId}/cleanup")
+public ResponseEntity<?> deleteDataLakeRetention(@PathVariable String
elementId) {
+ var measure = this.dataExplorerSchemaManagement.getById(elementId);
+ measure.deleteRetentionTime();
+ try {
+ this.dataExplorerSchemaManagement.updateMeasurement(measure);
+ } catch (IllegalArgumentException e) {
+ return badRequest(e.getMessage());
+ }
+ return ok();
+}
+
private ProvidedRestQueryParams populate(String measurementId, Map<String,
String> rawParams) {
Map<String, String> queryParamMap = new HashMap<>();
rawParams.forEach((key, value) -> queryParamMap.put(key, String.join(",",
value)));
diff --git a/streampipes-service-core/pom.xml b/streampipes-service-core/pom.xml
index 938af5f5c2..ccffcc14d8 100644
--- a/streampipes-service-core/pom.xml
+++ b/streampipes-service-core/pom.xml
@@ -109,6 +109,7 @@
<groupId>jakarta.annotation</groupId>
<artifactId>jakarta.annotation-api</artifactId>
</dependency>
+
<!-- Test dependencies -->
<dependency>
<groupId>org.junit.jupiter</groupId>
diff --git
a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DataLakeScheduler.java
b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DataLakeScheduler.java
index 753eb33950..61d08936c4 100644
---
a/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DataLakeScheduler.java
+++
b/streampipes-service-core/src/main/java/org/apache/streampipes/service/core/scheduler/DataLakeScheduler.java
@@ -17,31 +17,37 @@
*/
package org.apache.streampipes.service.core.scheduler;
+import org.apache.streampipes.commons.environment.Environments;
import org.apache.streampipes.dataexplorer.api.IDataExplorerQueryManagement;
import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
import
org.apache.streampipes.dataexplorer.export.ObjectStorge.ExportProviderFactory;
import org.apache.streampipes.dataexplorer.export.ObjectStorge.IObjectStorage;
import org.apache.streampipes.dataexplorer.export.OutputFormat;
import org.apache.streampipes.dataexplorer.management.DataExplorerDispatcher;
+import org.apache.streampipes.model.configuration.ExportProviderSettings;
+import org.apache.streampipes.model.configuration.ProviderType;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
-import org.apache.streampipes.model.datalake.ExportProviderSettings;
import org.apache.streampipes.model.datalake.RetentionAction;
import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
+import org.apache.streampipes.storage.management.StorageDispatcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.stereotype.Component;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.SchedulingConfigurer;
+import org.springframework.scheduling.config.ScheduledTaskRegistrar;
+import org.springframework.scheduling.support.CronTrigger;
import
org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
+import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-@Component
-public class DataLakeScheduler {
+@Configuration
+public class DataLakeScheduler implements SchedulingConfigurer {
private static final Logger LOG =
LoggerFactory.getLogger(DataLakeScheduler.class);
@@ -60,16 +66,18 @@ public class DataLakeScheduler {
}
var outputFormat = OutputFormat
-
.fromString(dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().format());
+
.fromString(dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig().format());
Map<String, String> params = new HashMap<>();
- params.put("delimiter",
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().csvDelimiter());
- params.put("format",
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().format());
+ params.put("delimiter",
+
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig().csvDelimiter());
+ params.put("format",
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig().format());
params.put("headerColumnName",
-
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().headerColumnName());
+
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig().headerColumnName());
params.put("missingValueBehaviour",
-
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().missingValueBehaviour());
+
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig()
+ .missingValueBehaviour());
params.put("endDate", Long.toString(endDate));
ProvidedRestQueryParams sanitizedParams = new
ProvidedRestQueryParams(dataLakeMeasure.getMeasureName(), params);
@@ -77,23 +85,60 @@ public class DataLakeScheduler {
sanitizedParams,
outputFormat,
"ignore".equals(
-
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().missingValueBehaviour()),
+
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig()
+ .missingValueBehaviour()),
output);
- try {
- ExportProviderSettings exportProviderSettings =
dataLakeMeasure.getRetentionTime().exportConfig()
- .exportProviderSettings();
- String providerType = exportProviderSettings.providerType();
+ String exportProviderId =
dataLakeMeasure.getRetentionTime().getRetentionExportConfig()
+ .getExportProviderId();
+ // FInd Item in Document
+
+ List<ExportProviderSettings> exportProviders =
StorageDispatcher.INSTANCE
+ .getNoSqlStore()
+ .getSpCoreConfigurationStorage()
+ .get()
+ .getExportProviderSettings();
+
+ ExportProviderSettings exportProviderSetting = null;
+
+ for (int i = 0; i < exportProviders.size(); i++) {
+ ExportProviderSettings existing = exportProviders.get(i);
+ if (existing != null &&
existing.getProviderId().equals(exportProviderId)) {
+ exportProviderSetting = existing;
+ }
+ }
- LOG.info("Write to " + System.getenv("SP_RETENTION_LOCAL_DIR"));
+ if (exportProviderSetting == null) {
+ LOG.error("The desired export provider was not found. No export
has been done.");
+ return;
+ }
+
+ ProviderType providerType = exportProviderSetting.getProviderType();
+
+ LOG.info("Write to " + System.getenv("SP_RETENTION_LOCAL_DIR"));
+
+ try {
IObjectStorage exportProvider =
ExportProviderFactory.createExportProvider(
- providerType, dataLakeMeasure.getMeasureName(),
exportProviderSettings,
-
dataLakeMeasure.getRetentionTime().exportConfig().exportConfig().format());
+ providerType, dataLakeMeasure.getMeasureName(),
exportProviderSetting,
+
dataLakeMeasure.getRetentionTime().getRetentionExportConfig().getExportConfig().format());
exportProvider.store(streamingOutput);
+ } catch (IllegalArgumentException e) {
+
+ LOG.error("Export provider could not be created. Unsupported
provider type: {}. Error: {}", providerType,
+ e.getMessage(), e);
+ } catch (IOException e) {
+
+ LOG.error("I/O error occurred while trying to store data. Provider
Type: {}. Error: {}", providerType,
+ e.getMessage(), e);
+ } catch (RuntimeException e) {
+ LOG.error("Runtime exception occurred while attempting to store
data. Provider Type: {}. Error: {}",
+ providerType, e.getMessage(), e);
} catch (Exception e) {
- e.printStackTrace();
+
+ LOG.error("An unexpected error occurred during export. Provider
Type: {}. Error: {}", providerType,
+ e.getMessage(), e);
}
}
@@ -117,8 +162,6 @@ public class DataLakeScheduler {
return result;
}
- @Scheduled(cron = "0 1 0 * * 6") // CronJob Scheduled every Saturday (6)
00:01 //@Scheduled(cron = "0 */2 * * *
- // *") //Cron Job in Dev Setting; Running
every 2 min
public void cleanupMeasurements() {
List<DataLakeMeasure> allMeasurements =
this.dataExplorerSchemaManagement.getAllMeasurements();
LOG.info("GET ALL Measurements");
@@ -127,16 +170,16 @@ public class DataLakeScheduler {
if (dataLakeMeasure.getRetentionTime() != null) {
var result = getStartAndEndTime(
-
dataLakeMeasure.getRetentionTime().dataRetentionConfig().olderThanDays());
+
dataLakeMeasure.getRetentionTime().getDataRetentionConfig().olderThanDays());
Instant now = (Instant) result.get("now");
long endDate = (Long) result.get("endDate");
- if
(dataLakeMeasure.getRetentionTime().dataRetentionConfig().action() !=
RetentionAction.DELETE) {
+ if
(dataLakeMeasure.getRetentionTime().getDataRetentionConfig().action() !=
RetentionAction.DELETE) {
LOG.info("Start saving Measurement " +
dataLakeMeasure.getMeasureName());
exportMeasurement(dataLakeMeasure, now, endDate);
LOG.info("Measurements " +
dataLakeMeasure.getMeasureName() + " successfully saved");
}
- if
(dataLakeMeasure.getRetentionTime().dataRetentionConfig().action() !=
RetentionAction.SAVE) {
+ if
(dataLakeMeasure.getRetentionTime().getDataRetentionConfig().action() !=
RetentionAction.SAVE) {
LOG.info("Start delete Measurement " +
dataLakeMeasure.getMeasureName());
deleteMeasurement(dataLakeMeasure, now, endDate);
LOG.info("Measurements " +
dataLakeMeasure.getMeasureName() + " successfully deleted");
@@ -144,4 +187,19 @@ public class DataLakeScheduler {
}
}
}
+
+ @Override
+ public void configureTasks(ScheduledTaskRegistrar taskRegistrar) {
+ var env = Environments.getEnvironment();
+ taskRegistrar.addTriggerTask(
+
+ this::cleanupMeasurements,
+
+
+ triggerContext -> new
CronTrigger(env.getDatalakeSchedulerCron().getValueOrDefault())
+ .nextExecution(triggerContext)
+
+ );
+
+ }
}
\ No newline at end of file
diff --git a/ui/deployment/i18n/de.json b/ui/deployment/i18n/de.json
index 840825bac3..30854ad724 100644
--- a/ui/deployment/i18n/de.json
+++ b/ui/deployment/i18n/de.json
@@ -32,9 +32,10 @@
"Stop pipeline": "Pipeline stoppen",
"Name": "Name",
"Last modified": "Zuletzt geändert",
- "Show pipeline": "Pipeline anzeigen",
- "Modify pipeline": "Pipeline ändern",
- "Permissions": "Berechtigungen",
+ "Show": "anzeigen",
+ "Edit": "Bearbeiten",
+ "Manage permissions": "Berechtigungen verwalten",
+ "Delete": "Löschen",
"no log messages available": "keine Logs verfügbar",
"Data Preview": "Datenvorschau",
"Enable live preview": "Live-Vorschau aktivieren",
@@ -49,6 +50,7 @@
"Start Pipeline": "Pipeline starten",
"Stop Pipeline": "Pipeline stoppen",
"Modify Pipeline": "Pipeline bearbeiten",
+ "Modify pipeline": "Pipeline ändern",
"Delete Pipeline": "Pipeline löschen",
"Preparing pipeline editor...": "Pipeline-Editor vorbereiten...",
"The tutorial requires pipeline elements that are not yet installed.": "Für
das Lernprogramm werden Pipeline-Elemente benötigt, die noch nicht installiert
sind.",
@@ -72,7 +74,6 @@
"Create new source": "Neue Quelle erstellen",
"Sort": "Sortieren",
"Group": "Gruppe",
- "Save Pipeline": "Pipeline speichern",
"Save pipeline": "Pipeline speichern",
"Auto Layout": "Auto-Layout",
"Add pipeline element": "Pipeline-Element hinzufügen",
@@ -192,9 +193,7 @@
"New chart": "Neues Diagramm",
"Chart": "Diagramm",
"Created": "Erstellt",
- "Show chart": "Diagramm anzeigen",
"Edit chart": "Diagramm bearbeiten",
- "Manage permissions": "Berechtigungen verwalten",
"Clone chart": "Diagramm kopieren",
"Delete chart": "Diagramm löschen",
"Chart Name": "Diagrammname",
@@ -254,11 +253,17 @@
"Default view mode": "Standard-Ansicht",
"Grid view": "Rasteransicht",
"Slide view": "Folienansicht",
+ "Grid": "Raster",
+ "Grid columns": "Gitternetz-Spalten",
"Time settings": "Zeiteinstellungen",
"Use global time settings instead of chart time settings": "Einheitliche
Zeiteinstellungen anstelle von Diagramm-Zeiteinstellungen verwenden",
+ "New dashboard title": "Neuer Dashboard-Titel",
+ "Clone options": "Optionen zum Klonen",
+ "Deep clone (also clone widgets)": "Tiefes Klonen (auch Widgets klonen)",
+ "Modify chart configurations": "Ändern von Diagrammkonfigurationen",
+ "Clone": "Klonen",
"This dashboard is empty and doesn't contain any charts.": "Dieses Dashboard
ist leer und hat keine Diagramm zum Anzeigen.",
"View mode": "Ansicht",
- "Grid": "Raster",
"Slides": "Folien",
"Options": "Optionen",
"Edit dashboard": "Dashboard bearbeiten",
@@ -270,9 +275,7 @@
"Create chart": "Diagramm erstellen",
"New dashboard": "Neues Dashboard",
"Dashboards": "Dashboards",
- "Show dashboard": "Dashboard anzeigen",
"Kiosk mode": "Kiosk-Modus",
- "Dashboard settings": "Dashboard-Einstellungen",
"ID": "ID",
"Output Topics": "Output-Topics",
"Copy": "Kopieren",
@@ -324,6 +327,27 @@
"Allow anonymous access through public link": "Anonymen Zugang über einen
öffentlichen Link ermöglichen",
"URL": "URL",
"(no log messages available)": "(keine Protokollmeldungen verfügbar)",
+ "New adapter": "Neuer Adapter",
+ "Start all adapters": "Alle Adapter starten",
+ "Stop all adapters": "Alle Adapter anhalten",
+ "Messages": "Nachrichten",
+ "Last message": "Letzte Nachricht",
+ "Add to Asset": "Zum Asset hinzufügen",
+ "Add Adapter to an existing Asset": "Adapter zu einem bestehenden Asset
hinzufügen",
+ "Remove Duplicates": "Duplikate entfernen",
+ "Avoid duplicated events within a certain time interval": "Vermeidung von
doppelten Ereignissen innerhalb eines bestimmten Zeitintervalls",
+ "Remove Duplicates Time Window": "Duplikate entfernen Zeitfenster",
+ "Reduce event rate": "Ereignisrate reduzieren",
+ "Send maximum one event in the specified time window": "Maximal ein Ereignis
im angegebenen Zeitfenster senden",
+ "Time Window (Milliseconds)": "Zeitfenster (Millisekunden)",
+ "Event Aggregation": "Ereignis-Aggregation",
+ "Persist events": "Ereignisse aufrechterhalten",
+ "Store all events of this source in the internal data store": "Speichern
aller Ereignisse dieser Quelle im internen Datenspeicher",
+ "Select Time Field": "Zeitfeld auswählen",
+ "Show code": "Code anzeigen",
+ "Show code to programmatically deploy this adapter over the API": "Code
anzeigen, um diesen Adapter programmatisch über die API einzusetzen",
+ "Start Adapter": "Start Adapter",
+ "Update Adapter": "Adapter aktualisieren",
"Sites & Areas": "Standorte & Bereiche",
"Manage your organization's sites and production areas": "Verwaltung der
Standorte und Produktionsbereiche",
"New site": "Neuer Standort",
@@ -352,12 +376,11 @@
"Group name": "Name der Benutzergruppe",
"Group ID": "Gruppen-ID",
"Edit user": "Benutzer bearbeiten",
- "Edit": "Bearbeiten",
"Delete service": "Service löschen",
- "Delete": "Löschen",
"New User": "Neuer Benutzer",
"Email": "E-Mail",
"Full Name": "Vor- und Nachname",
+ "Last Login": "Letzte Anmeldung",
"Delete user": "Benutzer löschen",
"New Service Account": "Neues Service-Konto",
"Username": "Benutzername",
@@ -455,7 +478,7 @@
"Issuer": "Aussteller",
"Expires": "Läuft aus",
"Certificate Details": "Zertifikat-Details",
- "Details": "Details",
+ "Details": "Einzelheiten",
"Reject": "Ablehnen",
"Trust": "Vertrauen",
"Delete certificate": "Zertifikat löschen",
@@ -548,16 +571,28 @@
"install": "installieren",
"uninstall": "deinstallieren",
"Press 'Next' to start the process.": "Auf \"Weiter\" drücken, um den
Vorgang zu starten.",
- "{{action}} {{current}} of {{total}} ({{name}})...{{status}}": "{{action}}
{{current}} von {{total}} ({{name}})...{{status}}",
+ "{{action}} {{ current }} of {{ total }} ({{ name }})...{{ status }}":
"{{Aktion}} {{ current }} von {{ total }} ({{ Name }})...{{ Status }}",
"Installing": "Installation von",
"Uninstalling": "Deinstallation von",
"More Details:": "Mehr Details:",
"Make available to all users with appropriate role": "Allen Benutzern mit
entsprechender Rolle zur Verfügung stellen",
- "Do you really want to delete the data index {{index}}?": "Den Datenindex
{{index}} wirklich löschen?",
- "Do you really want to truncate the data in {{index}}?": "Die Daten in
{{index}} wirklich leeren?",
+ "Select Provider": "Anbieter auswählen",
+ "S3": "S3",
+ "Access Key": "Access Key",
+ "Enter Access Key": "Access Key eingeben",
+ "Secret Key": "Secret Key",
+ "Enter Secret Key": "Secret Key eingeben",
+ "Endpoint": "Endpunkt",
+ "Enter Endpoint": "Endpunkt eingeben",
+ "Invalid URI format.": "Ungültiges URI-Format.",
+ "Bucket": "Bucket",
+ "Enter Bucket Name": "Bucketnamen eingeben",
+ "Region": "Region",
+ "Select Region": "Region auswählen",
"Truncate Data": "Daten leeren",
"Delete Data": "Daten löschen",
"Start Sync": "Synchronisierung starten",
+ "Delete Sync": "Sync löschen",
"Data Retention Action": "Aktion",
"delete": "löschen",
"save": "speichern",
@@ -568,8 +603,9 @@
"Export Settings": "Export-Einstellungen",
"Download Format": "Download-Format",
"Delimiter": "Trennzeichen",
- "Select Provider": "Anbieter auswählen",
- "local": "lokal",
+ "Export Provider": "Anbieter exportieren",
+ "Select Provider Type": "Anbietertyp auswählen",
+ "No export providers found. Please create one first.": "Keine Exportanbieter
gefunden. Bitte erstellen Sie zuerst einen.",
"Data Lake Settings": "Data Lake Einstellungen",
"Manage persisted data streams": "Verwalten von gespeicherten Datenströmen",
"Existing data lake indices": "Bestehende Data-Lake-Indizes",
@@ -584,6 +620,14 @@
"Retention Rate": "Speicherrichtlinie",
"Set retention rate": "Speicherrichtlinie festlegen",
"(no stored measurements)": "(keine gespeicherten Measurements)",
+ "Export Providers": "Exportanbieter",
+ "Add, Edit, and Delete export providers used for backing up data lakes.":
"Hinzufügen, Bearbeiten und Löschen von Exportanbietern, die für die Sicherung
von Data Lakes verwendet werden.",
+ "Existing Export Providers": "Bestehende Exportanbieter",
+ "New Export Provider": "Neuer Exportanbieter",
+ "Provider Type": "Anbieter Typ",
+ "Edit Export Provider": "Exportanbieter bearbeiten",
+ "Remove export provider configuration": "Konfiguration des Exportanbieters
entfernen",
+ "no stored export providers": "keine gespeicherten Exportanbieter",
"success": "Erfolg",
"error": "Fehler",
"waiting": "Warten",
@@ -625,7 +669,6 @@
"Save changes?": "Änderungen speichern?",
"Update all changes to chart or discard current changes.": "Aktualisieren
Sie alle Änderungen im Diagramm oder verwerfen Sie die aktuellen Änderungen.",
"Discard changes": "Änderungen verwerfen",
- "Update": "Aktualisieren",
"Millisecond": "Millisekunde",
"Second": "Sekunde",
"Minute": "Minute",
@@ -636,9 +679,9 @@
"Off": "Aus",
"Are you sure you want to delete this dashboard?": "Sind Sie sicher, dass
Sie dieses Dashboard löschen möchten?",
"This action cannot be undone!": "Diese Aktion kann nicht rückgängig gemacht
werden!",
+ "Clone dashboard": "Dashboard klonen",
"Topics": "Themen",
"Code": "Code",
- "Loading": "Laden",
"# Provide OPC UA Node IDs below, one per line.\n# Format:
ns=<namespace>;s=<node_id> (e.g., ns=3;s=SampleNodeId)\n": "# Geben Sie unten
OPC UA Node-IDs ein, eine pro Zeile.\n# Format: ns=<namespace>;s=<node_id> (z.
B. ns=3;s=SampleNodeId)\n",
"The value should be a number": "Der Wert sollte eine Zahl sein",
"Please enter a valid URL": "Bitte geben Sie eine gültige URL ein",
@@ -647,6 +690,11 @@
"This is a test": "Dies ist ein Test",
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not supported.":
"Fehler in Zeile {{rowNumber}}. Wert für \"{{property}}\" wird nicht
unterstützt.",
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not set.":
"Fehler in Zeile {{rowNumber}}. Wert für \"{{property}}\" ist nicht gesetzt.",
+ "Resources": "Ressourcen",
+ "Updating adapter {{adapterName}}": "Aktualisieren des Adapters
{{AdapterName}}",
+ "Creating adapter {{adapterName}}": "Adapter erstellen {{adapterName}}",
+ "Starting adapter {{adapterName}}": "Adapter starten {{AdapterName}}",
+ "Your {{assetTypes}} were successfully added to {{assetIds}}.": "Ihre
{{assetTypes}} wurden erfolgreich zu {{assetIds}} hinzugefügt.",
"General": "Allgemein",
"Data Lake": "Data Lake",
"Export/Import": "Export/Import",
@@ -720,7 +768,6 @@
"Refresh interval": "Aktualisierungsintervall",
"No entries available.": "Keine Einträge vorhanden.",
"Error": "Fehler",
- "Details": "Einzelheiten",
"Probable cause": "Wahrscheinliche Ursache",
"No more information": "Keine weiteren Informationen",
"Full details": "Alle Einzelheiten",
@@ -771,7 +818,6 @@
"Preview": "Vorschau",
"Documentation": "Dokumentation",
"Error Details": "Fehler-Details",
- "Resources": "Ressourcen",
"All {{allResourcesAlias}}": "Alle {{allResourcesAlias}}",
"{{ widgetTitle }} Clone": "{{ widgetTitle }} Kopie",
"{{action}} {{ current }} of {{ total }} ({{ name }})...{{ status }}":
"{{action}} {{ current }} von {{ total }} ({{name}})...{{ status }}"
diff --git a/ui/deployment/i18n/en.json b/ui/deployment/i18n/en.json
index 80d555a267..aa7fe2af2f 100644
--- a/ui/deployment/i18n/en.json
+++ b/ui/deployment/i18n/en.json
@@ -32,9 +32,10 @@
"Stop pipeline": null,
"Name": null,
"Last modified": null,
- "Show pipeline": null,
- "Modify pipeline": null,
- "Permissions": null,
+ "Show": null,
+ "Edit": null,
+ "Manage permissions": null,
+ "Delete": null,
"no log messages available": null,
"Data Preview": null,
"Enable live preview": null,
@@ -49,6 +50,7 @@
"Start Pipeline": null,
"Stop Pipeline": null,
"Modify Pipeline": null,
+ "Modify pipeline": null,
"Delete Pipeline": null,
"Preparing pipeline editor...": null,
"The tutorial requires pipeline elements that are not yet installed.": null,
@@ -72,7 +74,6 @@
"Create new source": null,
"Sort": null,
"Group": null,
- "Save Pipeline": null,
"Save pipeline": null,
"Auto Layout": null,
"Add pipeline element": null,
@@ -192,9 +193,7 @@
"New chart": null,
"Chart": null,
"Created": null,
- "Show chart": null,
"Edit chart": null,
- "Manage permissions": null,
"Clone chart": null,
"Delete chart": null,
"Chart Name": null,
@@ -254,11 +253,17 @@
"Default view mode": null,
"Grid view": null,
"Slide view": null,
+ "Grid": null,
+ "Grid columns": null,
"Time settings": null,
"Use global time settings instead of chart time settings": null,
+ "New dashboard title": null,
+ "Clone options": null,
+ "Deep clone (also clone widgets)": null,
+ "Modify chart configurations": null,
+ "Clone": null,
"This dashboard is empty and doesn't contain any charts.": null,
"View mode": null,
- "Grid": null,
"Slides": null,
"Options": null,
"Edit dashboard": null,
@@ -270,9 +275,7 @@
"Create chart": null,
"New dashboard": null,
"Dashboards": null,
- "Show dashboard": null,
"Kiosk mode": null,
- "Dashboard settings": null,
"ID": null,
"Output Topics": null,
"Copy": null,
@@ -324,6 +327,27 @@
"Allow anonymous access through public link": null,
"URL": null,
"(no log messages available)": null,
+ "New adapter": null,
+ "Start all adapters": null,
+ "Stop all adapters": null,
+ "Messages": null,
+ "Last message": null,
+ "Add to Asset": null,
+ "Add Adapter to an existing Asset": null,
+ "Remove Duplicates": null,
+ "Avoid duplicated events within a certain time interval": null,
+ "Remove Duplicates Time Window": null,
+ "Reduce event rate": null,
+ "Send maximum one event in the specified time window": null,
+ "Time Window (Milliseconds)": null,
+ "Event Aggregation": null,
+ "Persist events": null,
+ "Store all events of this source in the internal data store": null,
+ "Select Time Field": null,
+ "Show code": null,
+ "Show code to programmatically deploy this adapter over the API": null,
+ "Start Adapter": null,
+ "Update Adapter": null,
"Sites & Areas": null,
"Manage your organization's sites and production areas": null,
"New site": null,
@@ -352,12 +376,11 @@
"Group name": null,
"Group ID": null,
"Edit user": null,
- "Edit": null,
"Delete service": null,
- "Delete": null,
"New User": null,
"Email": null,
"Full Name": null,
+ "Last Login": null,
"Delete user": null,
"New Service Account": null,
"Username": null,
@@ -463,9 +486,9 @@
"Uninstall selected": null,
"All": null,
"Adapters": null,
- "Streams": null,
+ "Data Streams": null,
"Data Processors": null,
- "Sinks": null,
+ "Data Sinks": null,
"Find Element": null,
"Available & Installed": null,
"Installed": null,
@@ -490,7 +513,6 @@
"Upload application package file": null,
"Select resources to import": null,
"Assets": null,
- "Data Streams": null,
"Data Lake Storage": null,
"Pipelines": null,
"Import options": null,
@@ -531,7 +553,7 @@
"You can set various placeholder variables that will be replaced with the
actual values when sending an email:": null,
"Save changes": null,
"Choose a name for your site": null,
- "Site label is required": null,
+ "Site name is required": null,
"Available areas within the site (e.g. plants or facilities)": null,
"Location": null,
"Exact location of the site": null,
@@ -549,16 +571,28 @@
"install": null,
"uninstall": null,
"Press 'Next' to start the process.": null,
- "{{action}} {{current}} of {{total}} ({{name}})...{{status}}": "{{action}}
{{current}} of {{total}} ({{name}})...{{status}}",
+ "{{action}} {{ current }} of {{ total }} ({{ name }})...{{ status }}": null,
"Installing": null,
"Uninstalling": null,
"More Details:": null,
"Make available to all users with appropriate role": null,
- "Do you really want to delete the data index {{index}}?": "Do you really
want to delete the data index {{index}}?",
- "Do you really want to truncate the data in {{index}}?": "Do you really want
to truncate the data in {{index}}?",
+ "Select Provider": null,
+ "S3": null,
+ "Access Key": null,
+ "Enter Access Key": null,
+ "Secret Key": null,
+ "Enter Secret Key": null,
+ "Endpoint": null,
+ "Enter Endpoint": null,
+ "Invalid URI format.": null,
+ "Bucket": null,
+ "Enter Bucket Name": null,
+ "Region": null,
+ "Select Region": null,
"Truncate Data": null,
"Delete Data": null,
"Start Sync": null,
+ "Delete Sync": null,
"Data Retention Action": null,
"delete": null,
"save": null,
@@ -569,8 +603,9 @@
"Export Settings": null,
"Download Format": null,
"Delimiter": null,
- "Select Provider": null,
- "local": null,
+ "Export Provider": null,
+ "Select Provider Type": null,
+ "No export providers found. Please create one first.": null,
"Data Lake Settings": null,
"Manage persisted data streams": null,
"Existing data lake indices": null,
@@ -585,6 +620,14 @@
"Retention Rate": null,
"Set retention rate": null,
"(no stored measurements)": null,
+ "Export Providers": null,
+ "Add, Edit, and Delete export providers used for backing up data lakes.":
null,
+ "Existing Export Providers": null,
+ "New Export Provider": null,
+ "Provider Type": null,
+ "Edit Export Provider": null,
+ "Remove export provider configuration": null,
+ "no stored export providers": null,
"success": null,
"error": null,
"waiting": null,
@@ -626,7 +669,6 @@
"Save changes?": null,
"Update all changes to chart or discard current changes.": null,
"Discard changes": null,
- "Update": null,
"Millisecond": null,
"Second": null,
"Minute": null,
@@ -637,9 +679,9 @@
"Off": null,
"Are you sure you want to delete this dashboard?": null,
"This action cannot be undone!": null,
+ "Clone dashboard": null,
"Topics": null,
"Code": null,
- "Loading": null,
"# Provide OPC UA Node IDs below, one per line.\n# Format:
ns=<namespace>;s=<node_id> (e.g., ns=3;s=SampleNodeId)\n": null,
"The value should be a number": null,
"Please enter a valid URL": null,
@@ -648,6 +690,11 @@
"This is a test": null,
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not supported.":
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not supported.",
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not set.":
"Error in line {{rowNumber}}. Value for \"{{property}}\" is not set.",
+ "Resources": null,
+ "Updating adapter {{adapterName}}": "Updating adapter {{adapterName}}",
+ "Creating adapter {{adapterName}}": "Creating adapter {{adapterName}}",
+ "Starting adapter {{adapterName}}": "Starting adapter {{adapterName}}",
+ "Your {{assetTypes}} were successfully added to {{assetIds}}.": "Your
{{assetTypes}} were successfully added to {{assetIds}}.",
"General": null,
"Data Lake": null,
"Export/Import": null,
@@ -717,12 +764,10 @@
"Ignore lines with missing value": null,
"Leave entry empty": null,
"Download successful": null,
- "Refresh": null,
"Apply": null,
"Refresh interval": null,
"No entries available.": null,
"Error": null,
- "Details": null,
"Probable cause": null,
"No more information": null,
"Full details": null,
@@ -773,7 +818,6 @@
"Preview": null,
"Documentation": null,
"Error Details": null,
- "Resources": null,
"All {{allResourcesAlias}}": "All {{allResourcesAlias}}",
"{{ widgetTitle }} Clone": "{{ widgetTitle }} Clone",
"Your {{assetTypes}} were successfully added to {{assetIds}}.": "Your
{{assetTypes}} were successfully added to {{assetIds}}.",
diff --git
a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index fc47f133ea..e5eeb66950 100644
---
a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -163,7 +163,6 @@ export class DatalakeRestService {
cleanup(index: string, config: any) {
const url = `${this.dataLakeUrl}/${index}/cleanup`;
-
const request = new HttpRequest('POST', url, config, {
headers: new HttpHeaders({ 'Content-Type': 'application/json' }),
// optional if already handled globally
});
@@ -171,6 +170,11 @@ export class DatalakeRestService {
return this.http.request(request);
}
+ deleteCleanup(index: string) {
+ const url = `${this.dataLakeUrl}/${index}/cleanup`;
+ return this.http.delete(url);
+ }
+
buildDownloadRequest(index: string, queryParams: any) {
const url = this.dataLakeUrl + '/measurements/' + index + '/download';
const request = new HttpRequest('GET', url, {
diff --git
a/ui/projects/streampipes/platform-services/src/lib/apis/export-provider.service.ts
b/ui/projects/streampipes/platform-services/src/lib/apis/export-provider.service.ts
new file mode 100644
index 0000000000..b59b57d55c
--- /dev/null
+++
b/ui/projects/streampipes/platform-services/src/lib/apis/export-provider.service.ts
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { inject, Injectable } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { Observable } from 'rxjs';
+import { PlatformServicesCommons } from './commons.service';
+import { ExportProviderSettings } from '@streampipes/platform-services';
+
+@Injectable({
+ providedIn: 'root',
+})
+export class ExportProviderService {
+ private http = inject(HttpClient);
+ private platformServicesCommons = inject(PlatformServicesCommons);
+
+ getAllExportProviders(): Observable<ExportProviderSettings[]> {
+ return this.http.get<ExportProviderSettings[]>(
+ this.exportProviderBasePath,
+ );
+ }
+ getExportProviderById(
+ providerId: string,
+ ): Observable<ExportProviderSettings> {
+ return this.http.get<ExportProviderSettings>(
+ `${this.exportProviderBasePath}/${providerId}`,
+ );
+ }
+
+ updateExportProvider(
+ exportProviderSettings: ExportProviderSettings,
+ ): Observable<ExportProviderSettings> {
+ return this.http.put<ExportProviderSettings>(
+ `${this.exportProviderBasePath}`,
+ exportProviderSettings,
+ );
+ }
+
+ deleteExportProvider(exportProviderId: string): Observable<void> {
+ return this.http.delete<void>(
+ `${this.exportProviderBasePath}/${exportProviderId}`,
+ );
+ }
+
+ private get exportProviderBasePath() {
+ return (
+ this.platformServicesCommons.apiBasePath +
+ '/admin/exportprovider-config'
+ );
+ }
+}
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
index 47ae72ca3e..55adefc997 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model-client.ts
@@ -16,13 +16,12 @@
* specific language governing permissions and limitations
* under the License.
*/
-
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2025-10-01
08:36:56.
+// Generated using typescript-generator version 3.2.1263 on 2025-10-09
16:21:15.
-import { Storable } from './streampipes-model';
+import { Storable } from './platform-services';
export class Group implements Storable {
alternateIds: string[];
diff --git
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
index 6823c795ec..ebe29936c7 100644
---
a/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/model/gen/streampipes-model.ts
@@ -16,11 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-
/* tslint:disable */
/* eslint-disable */
// @ts-nocheck
-// Generated using typescript-generator version 3.2.1263 on 2025-10-01
08:36:50.
+// Generated using typescript-generator version 3.2.1263 on 2025-10-09
16:21:10.
export class NamedStreamPipesEntity implements Storable {
'@class':
@@ -1162,6 +1161,7 @@ export class DashboardModel implements Storable,
SpResource {
description: string;
displayHeader: boolean;
elementId: string;
+ gridColumns: number;
id: string;
metadata: ResourceMetadata;
name: string;
@@ -1189,6 +1189,7 @@ export class DashboardModel implements Storable,
SpResource {
instance.description = data.description;
instance.displayHeader = data.displayHeader;
instance.elementId = data.elementId;
+ instance.gridColumns = data.gridColumns;
instance.id = data.id;
instance.metadata = ResourceMetadata.fromData(data.metadata);
instance.name = data.name;
@@ -1875,7 +1876,14 @@ export class ExportItem {
}
export class ExportProviderSettings {
- providerType: string;
+ accessKey: string;
+ awsRegion: string;
+ bucketName: string;
+ endPoint: string;
+ providerId: string;
+ providerType: ProviderType;
+ secretEncrypted: boolean;
+ secretKey: string;
static fromData(
data: ExportProviderSettings,
@@ -1885,7 +1893,14 @@ export class ExportProviderSettings {
return data;
}
const instance = target || new ExportProviderSettings();
+ instance.accessKey = data.accessKey;
+ instance.awsRegion = data.awsRegion;
+ instance.bucketName = data.bucketName;
+ instance.endPoint = data.endPoint;
+ instance.providerId = data.providerId;
instance.providerType = data.providerType;
+ instance.secretEncrypted = data.secretEncrypted;
+ instance.secretKey = data.secretKey;
return instance;
}
}
@@ -3356,7 +3371,7 @@ export class ResourceMetadata {
export class RetentionExportConfig {
exportConfig: ExportConfig;
- exportProviderSettings: ExportProviderSettings;
+ exportProviderId: string;
static fromData(
data: RetentionExportConfig,
@@ -3367,16 +3382,14 @@ export class RetentionExportConfig {
}
const instance = target || new RetentionExportConfig();
instance.exportConfig = ExportConfig.fromData(data.exportConfig);
- instance.exportProviderSettings = ExportProviderSettings.fromData(
- data.exportProviderSettings,
- );
+ instance.exportProviderId = data.exportProviderId;
return instance;
}
}
export class RetentionTimeConfig {
dataRetentionConfig: DataRetentionConfig;
- exportConfig: RetentionExportConfig;
+ retentionExportConfig: RetentionExportConfig;
static fromData(
data: RetentionTimeConfig,
@@ -3389,8 +3402,8 @@ export class RetentionTimeConfig {
instance.dataRetentionConfig = DataRetentionConfig.fromData(
data.dataRetentionConfig,
);
- instance.exportConfig = RetentionExportConfig.fromData(
- data.exportConfig,
+ instance.retentionExportConfig = RetentionExportConfig.fromData(
+ data.retentionExportConfig,
);
return instance;
}
@@ -4301,6 +4314,8 @@ export type PropertyScope =
| 'MEASUREMENT_PROPERTY'
| 'NONE';
+export type ProviderType = 'FOLDER' | 'S3';
+
export type RetentionAction = 'DELETE' | 'SAVE' | 'SAVEDELETE';
export type RetentionInterval = 'DAILY' | 'MONTHLY' | 'WEEKLY';
diff --git a/ui/src/app/configuration/configuration.module.ts
b/ui/src/app/configuration/configuration.module.ts
index 51f8856b7d..f4ffeaafec 100644
--- a/ui/src/app/configuration/configuration.module.ts
+++ b/ui/src/app/configuration/configuration.module.ts
@@ -107,10 +107,12 @@ import { UserAcknowledgmentComponent } from
'./general-configuration/user-acknow
import { QuillEditorComponent } from 'ngx-quill';
import { MatStepperModule } from '@angular/material/stepper';
import { DataRetentionDialogComponent } from
'./dialog/data-retention-dialog/data-retention-dialog.component';
+import { ExportProviderComponent } from
'./dialog/export-provider-dialog/export-provider-dialog.component';
import { SelectDataComponent } from
'./dialog/data-retention-dialog/components/select-retention/select-data.component';
import { SelectDataRetentionComponent } from
'./dialog/data-retention-dialog/components/select-retention/select-data-retention/select-data-retention.component';
import { SelectRetentionActionComponent } from
'./dialog/data-retention-dialog/components/select-retention/select-retention-action/select-retention-action.component';
import { SelectDataExportComponent } from
'./dialog/data-retention-dialog/components/select-export/select-format.component';
+import { DeleteExportProviderComponent } from
'./dialog/delete-export-provider/delete-export-provider-dialog.component';
@NgModule({
imports: [
CommonModule,
@@ -261,6 +263,7 @@ import { SelectDataExportComponent } from
'./dialog/data-retention-dialog/compon
SpRegisteredExtensionsServiceComponent,
SpExtensionsServiceConfigurationComponent,
DataRetentionDialogComponent,
+ ExportProviderComponent,
SpExtensionsInstallationComponent,
SpExtensionsInstallationDialogComponent,
EndpointItemComponent,
@@ -276,6 +279,7 @@ import { SelectDataExportComponent } from
'./dialog/data-retention-dialog/compon
SelectDataRetentionComponent,
SelectRetentionActionComponent,
SelectDataExportComponent,
+ DeleteExportProviderComponent,
],
providers: [
OrderByPipe,
diff --git
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
index c5bf509434..52fa1c9191 100644
---
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
+++
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.html
@@ -279,6 +279,203 @@
<h5>{{ '(no stored measurements)' | translate }}</h5>
}
</sp-split-section>
+ <sp-split-section
+ [title]="'Export Providers' | translate"
+ [subtitle]="
+ 'Add, Edit, and Delete export providers used for backing
up data lakes.'
+ | translate
+ "
+ >
+ <div fxFlex="100" fxLayout="column">
+ <div fxFlex="100" fxLayout="row">
+ <div
+ fxFlex
+ fxLayoutAlign="start center"
+ class="subsection-title"
+ >
+ {{ 'Existing Export Providers' | translate }}
+ </div>
+
+ <div fxLayout="row" fxLayoutAlign="end center">
+ <button
+ color="accent"
+ mat-icon-button
+ matTooltip="{{
+ 'New Export Provider' | translate
+ }}"
+ data-cy="new-export-providers"
+ (click)="createExportProvider(null)"
+ >
+ <mat-icon>add</mat-icon>
+ </button>
+ </div>
+
+ <div fxLayout="row" fxLayoutAlign="end center">
+ <button
+ color="accent"
+ mat-icon-button
+ matTooltip="{{ 'Refresh' | translate }}"
+ data-cy="refresh-export-providers-measures"
+ (click)="loadAvailableExportProvider()"
+ >
+ <mat-icon>refresh</mat-icon>
+ </button>
+ </div>
+ </div>
+
+ <table
+ fxFlex="100"
+ mat-table
+ data-cy="exportproviders-settings"
+ [dataSource]="dataSourceExport"
+ style="width: 100%"
+ matSort
+ >
+ <ng-container matColumnDef="providertype">
+ <th
+ mat-header-cell
+ mat-sort-header
+ *matHeaderCellDef
+ >
+ {{ 'Provider Type' | translate }}
+ </th>
+ <td mat-cell *matCellDef="let configurationEntry">
+ <h4 style="margin-bottom: 0px">
+ {{ configurationEntry.providerType }}
+ </h4>
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="endpoint">
+ <th
+ mat-header-cell
+ mat-sort-header
+ *matHeaderCellDef
+ >
+ {{ 'Endpoint' | translate }}
+ </th>
+ <td mat-cell *matCellDef="let configurationEntry">
+ {{ configurationEntry.endPoint }}
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="bucket">
+ <th
+ mat-header-cell
+ mat-sort-header
+ *matHeaderCellDef
+ >
+ {{ 'Bucket' | translate }}
+ </th>
+ <td mat-cell *matCellDef="let configurationEntry">
+ {{ configurationEntry.bucketName }}
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="editExportProvider">
+ <th mat-header-cell *matHeaderCellDef>
+ {{ 'Edit' | translate }}
+ </th>
+ <td mat-cell *matCellDef="let configurationEntry">
+ <div
+ *ngIf="
+ configurationEntry.providerId !==
+ 'FOLDER'
+ "
+ >
+ <div fxLayout="row">
+ <span
+ fxFlex
+ fxFlexOrder="3"
+ fxLayout="row"
+ fxLayoutAlign="start center"
+ >
+ <button
+ color="accent"
+ mat-icon-button
+ matTooltip="{{
+ 'Edit Export Provider'
+ | translate
+ }}"
+ matTooltipPosition="above"
+
data-cy="exportProvider-edit-btn"
+ (click)="
+ createExportProvider(
+ configurationEntry
+ )
+ "
+ >
+ <i class="material-icons"
+ >edit</i
+ >
+ </button>
+ </span>
+ </div>
+ </div>
+ </td>
+ </ng-container>
+
+ <ng-container matColumnDef="delete">
+ <th mat-header-cell *matHeaderCellDef>
+ {{ 'Remove' | translate }}
+ </th>
+ <td mat-cell *matCellDef="let configurationEntry">
+ <div
+ *ngIf="
+ configurationEntry.providerId !==
+ 'FOLDER'
+ "
+ >
+ <div fxLayout="row">
+ <span
+ fxFlex
+ fxFlexOrder="3"
+ fxLayout="row"
+ fxLayoutAlign="start center"
+ >
+ <button
+ color="accent"
+ mat-icon-button
+ matTooltip="{{
+ 'Remove export provider
configuration'
+ | translate
+ }}"
+
data-cy="exportProvider-delete-btn"
+ matTooltipPosition="above"
+ (click)="
+ deleteExportProvider(
+
configurationEntry.providerId
+ )
+ "
+ >
+ <i class="material-icons"
+ >delete</i
+ >
+ </button>
+ </span>
+ </div>
+ </div>
+ </td>
+ </ng-container>
+
+ <tr
+ mat-header-row
+ *matHeaderRowDef="displayedColumnsExport"
+ ></tr>
+ <tr
+ mat-row
+ *matRowDef="
+ let row;
+ columns: displayedColumnsExport
+ "
+ ></tr>
+ </table>
+ </div>
+
+ <div *ngIf="availableExportProvider.length === 0">
+ <h5>{{ 'no stored export providers' | translate }}</h5>
+ </div>
+ </sp-split-section>
</div>
</div>
</sp-basic-nav-tabs>
diff --git
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts
index ef79e9ed71..80a0c30f28 100644
---
a/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts
+++
b/ui/src/app/configuration/datalake-configuration/datalake-configuration.component.ts
@@ -16,12 +16,13 @@
*
*/
-import { Component, OnInit, ViewChild } from '@angular/core';
+import { Component, inject, OnInit, ViewChild } from '@angular/core';
import { MatTableDataSource } from '@angular/material/table';
import { DataLakeConfigurationEntry } from './datalake-configuration-entry';
import {
ChartService,
DatalakeRestService,
+ ExportProviderSettings,
} from '@streampipes/platform-services';
import { MatPaginator } from '@angular/material/paginator';
import { MatSort } from '@angular/material/sort';
@@ -37,6 +38,9 @@ import { DeleteDatalakeIndexComponent } from
'../dialog/delete-datalake-index/de
import { SpConfigurationTabsService } from '../configuration-tabs.service';
import { SpConfigurationRoutes } from '../configuration.routes';
import { DataRetentionDialogComponent } from
'../dialog/data-retention-dialog/data-retention-dialog.component';
+import { ExportProviderService } from
'projects/streampipes/platform-services/src/lib/apis/export-provider.service';
+import { ExportProviderComponent } from
'../dialog/export-provider-dialog/export-provider-dialog.component';
+import { DeleteExportProviderComponent } from
'../dialog/delete-export-provider/delete-export-provider-dialog.component';
import { TranslateService } from '@ngx-translate/core';
@Component({
@@ -51,9 +55,21 @@ export class DatalakeConfigurationComponent implements
OnInit {
@ViewChild(MatPaginator) paginator: MatPaginator;
@ViewChild(MatSort) sort: MatSort;
+ private datalakeRestService = inject(DatalakeRestService);
+ private dataViewDataExplorerService = inject(ChartService);
+ private dialogService = inject(DialogService);
+ private breadcrumbService = inject(SpBreadcrumbService);
+ private tabService = inject(SpConfigurationTabsService);
+ private exportProviderRestService = inject(ExportProviderService);
+ private translateService = inject(TranslateService);
+
dataSource: MatTableDataSource<DataLakeConfigurationEntry> =
new MatTableDataSource([]);
availableMeasurements: DataLakeConfigurationEntry[] = [];
+ availableExportProvider: ExportProviderSettings[] = [];
+
+ dataSourceExport: MatTableDataSource<ExportProviderSettings> =
+ new MatTableDataSource([]);
displayedColumns: string[] = [
'name',
@@ -65,18 +81,17 @@ export class DatalakeConfigurationComponent implements
OnInit {
'retention',
];
+ displayedColumnsExport: string[] = [
+ 'providertype',
+ 'endpoint',
+ 'bucket',
+ 'editExportProvider',
+ 'delete',
+ ];
+
pageSize = 15;
pageIndex = 0;
- constructor(
- private datalakeRestService: DatalakeRestService,
- private dataViewDataExplorerService: ChartService,
- private dialogService: DialogService,
- private breadcrumbService: SpBreadcrumbService,
- private tabService: SpConfigurationTabsService,
- private translateService: TranslateService,
- ) {}
-
ngOnInit(): void {
this.tabs = this.tabService.getTabs();
this.breadcrumbService.updateBreadcrumb([
@@ -84,6 +99,17 @@ export class DatalakeConfigurationComponent implements
OnInit {
{ label: this.tabService.getTabTitle('datalake') },
]);
this.loadAvailableMeasurements();
+ this.loadAvailableExportProvider();
+ }
+
+ loadAvailableExportProvider() {
+ this.availableExportProvider = [];
+ this.exportProviderRestService
+ .getAllExportProviders()
+ .subscribe(allExportProviders => {
+ this.availableExportProvider = allExportProviders;
+ this.dataSourceExport.data = this.availableExportProvider;
+ });
}
loadAvailableMeasurements() {
@@ -133,6 +159,22 @@ export class DatalakeConfigurationComponent implements
OnInit {
});
}
+ createExportProvider(provider: ExportProviderSettings | null) {
+ const dialogRef: DialogRef<ExportProviderComponent> =
+ this.dialogService.open(ExportProviderComponent, {
+ panelType: PanelType.STANDARD_PANEL,
+ title: 'New Export Provider',
+ width: '70vw',
+ data: {
+ provider: provider,
+ },
+ });
+
+ dialogRef.afterClosed().subscribe(() => {
+ this.loadAvailableExportProvider();
+ });
+ }
+
cleanDatalakeIndex(measurementIndex: string) {
const dialogRef: DialogRef<DeleteDatalakeIndexComponent> =
this.dialogService.open(DeleteDatalakeIndexComponent, {
@@ -171,6 +213,24 @@ export class DatalakeConfigurationComponent implements
OnInit {
});
}
+ deleteExportProvider(providerId: string) {
+ const dialogRef: DialogRef<DeleteExportProviderComponent> =
+ this.dialogService.open(DeleteExportProviderComponent, {
+ panelType: PanelType.STANDARD_PANEL,
+ title: 'Delete Export Provider',
+ width: '70vw',
+ data: {
+ providerId: providerId,
+ },
+ });
+
+ dialogRef.afterClosed().subscribe(data => {
+ if (data) {
+ this.loadAvailableExportProvider();
+ }
+ });
+ }
+
openDownloadDialog(measurementName: string) {
this.dialogService.open(DataDownloadDialogComponent, {
panelType: PanelType.SLIDE_IN_PANEL,
diff --git
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.html
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.html
index 54f7253134..0a98cce1c9 100644
---
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.html
+++
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.html
@@ -22,7 +22,8 @@
<mat-radio-group
class="sp-radio-group"
[(ngModel)]="
- dataRetentionConfig.exportConfig.exportConfig.format
+ dataRetentionConfig.retentionExportConfig.exportConfig
+ .format
"
>
<mat-radio-button
@@ -50,11 +51,14 @@
</sp-configuration-box>
</div>
- @if (dataRetentionConfig.exportConfig.exportConfig.format === 'csv') {
+ @if (
+ dataRetentionConfig.retentionExportConfig.exportConfig.format === 'csv'
+ ) {
<sp-configuration-box [title]="'Delimiter' | translate">
<mat-radio-group
[(ngModel)]="
- dataRetentionConfig.exportConfig.exportConfig.delimiter
+ dataRetentionConfig.retentionExportConfig.exportConfig
+ .csvDelimiter
"
class="sp-radio-group"
>
@@ -73,18 +77,71 @@
</mat-radio-group>
</sp-configuration-box>
}
- <mat-form-field appearance="fill" class="sp-select-field">
- <mat-label>{{ 'Select Provider' | translate }}</mat-label>
- <mat-select
- [(ngModel)]="
- dataRetentionConfig.exportConfig.exportProviderSettings
- .providerType
- "
- data-cy="exportProvider-select"
- >
- <mat-option value="local" data-cy="exportProvider-local">
- {{ 'local' | translate }}
- </mat-option>
- </mat-select>
- </mat-form-field>
+ <sp-configuration-box [title]="'Export Provider' | translate">
+ <div class="mt-10 p-10">
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Select Provider Type' | translate }}</mat-label>
+ <mat-select
+ [(ngModel)]="selectedProviderType"
+ data-cy="exportProvider-select"
+ (selectionChange)="onProviderTypeChange($event.value)"
+ >
+ @for (type of providerType; track type) {
+ <mat-option
+ [value]="type.toUpperCase()"
+ [attr.data-cy]="
+ 'exportProvider-' + type.toLowerCase()
+ "
+ >
+ {{ type | translate }}
+ </mat-option>
+ }
+ </mat-select>
+ </mat-form-field>
+ @if (selectedProviderType === 'S3') {
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Select Provider' | translate }}</mat-label>
+ <mat-select
+ [(ngModel)]="
+ dataRetentionConfig.retentionExportConfig
+ .exportProviderId
+ "
+ data-cy="exportProviderId-select"
+ >
+ @for (
+ provider of availableS3ExportProvider;
+ track provider
+ ) {
+ <mat-option
+ [value]="provider.providerId"
+ [attr.data-cy]="
+ 'exportProviderId-' + provider.providerId
+ "
+ style="font-size: 12px"
+ >
+ {{
+ 'Endpoint: ' +
+ provider.endPoint +
+ ' Bucket: ' +
+ provider.bucketName
+ }}
+ </mat-option>
+ }
+ </mat-select>
+ </mat-form-field>
+ }
+ </div>
+ @if (
+ selectedProviderType === 'S3' &&
+ availableS3ExportProvider?.length === 0
+ ) {
+ <div fxLayout="row" fxLayoutAlign="start center">
+ <mat-icon color="primary" class="mr-5
info-icon">info</mat-icon>
+ {{
+ 'No export providers found. Please create one first.'
+ | translate
+ }}
+ </div>
+ }
+ </sp-configuration-box>
</sp-configuration-box>
diff --git
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.scss
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.scss
index 2b53bbd279..a7bb6246ac 100644
---
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.scss
+++
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.scss
@@ -15,3 +15,22 @@
* limitations under the License.
*
*/
+.invalid-uri {
+ border: var(--color-warn);
+}
+
+.s3-config-container {
+ margin-top: 5px;
+ padding: 15px;
+ background-color: var(--color-bg-2);
+ border-radius: 8px;
+}
+
+.s3-config-container mat-form-field {
+ margin-bottom: 5px;
+}
+
+mat-hint {
+ color: red;
+ font-size: 12px;
+}
diff --git
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.ts
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.ts
index 7de468301f..c6b8a54c17 100644
---
a/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.ts
+++
b/ui/src/app/configuration/dialog/data-retention-dialog/components/select-export/select-format.component.ts
@@ -16,9 +16,13 @@
*
*/
-import { Component, Input, OnInit } from '@angular/core';
-import { DataExplorerDataConfig } from '@streampipes/platform-services';
+import { Component, inject, Input, OnInit } from '@angular/core';
+import {
+ DataExplorerDataConfig,
+ ExportProviderSettings,
+} from '@streampipes/platform-services';
import { RetentionTimeConfig } from '@streampipes/platform-services';
+import { ExportProviderService } from
'projects/streampipes/platform-services/src/lib/apis/export-provider.service';
@Component({
selector: 'sp-data-export',
@@ -26,9 +30,92 @@ import { RetentionTimeConfig } from
'@streampipes/platform-services';
styleUrls: ['./select-format.component.scss'],
standalone: false,
})
-export class SelectDataExportComponent {
+export class SelectDataExportComponent implements OnInit {
@Input()
dataExplorerDataConfig: DataExplorerDataConfig;
@Input()
dataRetentionConfig: RetentionTimeConfig;
+
+ exportProviderRestService = inject(ExportProviderService);
+
+ exportProvider: ExportProviderSettings;
+
+ availableExportProvider: ExportProviderSettings[] = [];
+ availableS3ExportProvider: ExportProviderSettings[] = [];
+ availableFolderExportProvider: ExportProviderSettings[] = [];
+ providerType: string[] = ['Folder', 'S3'];
+ selectedProviderType: string;
+ selectedProviderId: string;
+
+ ngOnInit() {
+ this.loadAvailableExportProvider();
+ this.selectedProviderType = 'FOLDER';
+
+ if (
+ this.dataRetentionConfig.retentionExportConfig.exportProviderId !==
+ ''
+ ) {
+ this.exportProviderRestService
+ .getExportProviderById(
+ this.dataRetentionConfig.retentionExportConfig
+ .exportProviderId,
+ )
+ .subscribe(exportProvider => {
+ this.exportProvider = exportProvider;
+ if (this.exportProvider) {
+ this.selectedProviderType =
+ this.exportProvider.providerType;
+
+ this.selectedProviderId =
+ this.exportProvider.providerId;
+
this.dataRetentionConfig.retentionExportConfig.exportProviderId =
+ this.selectedProviderId;
+ }
+ });
+ }
+ }
+
+ loadAvailableExportProvider() {
+ this.availableExportProvider = [];
+ this.exportProviderRestService
+ .getAllExportProviders()
+ .subscribe(allExportProviders => {
+ this.availableExportProvider = allExportProviders;
+
+ this.availableS3ExportProvider =
+ this.availableExportProvider.filter(
+ provider => provider.providerType === 'S3',
+ );
+ this.availableFolderExportProvider =
+ this.availableExportProvider.filter(
+ provider => provider.providerType === 'FOLDER',
+ );
+ // Defualts to Folder
+
this.dataRetentionConfig.retentionExportConfig.exportProviderId =
+ this.availableFolderExportProvider[0].providerId;
+ });
+ }
+ onProviderTypeChange(type: string): void {
+ this.selectedProviderType = type;
+ // sets default
+ if (
+ type === 'FOLDER' &&
+ this.availableFolderExportProvider.length > 0
+ ) {
+ this.dataRetentionConfig.retentionExportConfig.exportProviderId =
+ this.availableFolderExportProvider[0].providerId;
+ this.selectedProviderId =
+ this.availableFolderExportProvider[0].providerId;
+ } else if (type === 'S3' && this.availableS3ExportProvider.length > 0)
{
+ this.dataRetentionConfig.retentionExportConfig.exportProviderId =
+ this.availableS3ExportProvider[0].providerId;
+ this.selectedProviderId =
+ this.availableS3ExportProvider[0].providerId;
+ } else {
+ // no providers available for this type, clear the exportProviderId
+ this.dataRetentionConfig.retentionExportConfig.exportProviderId =
+ '';
+ this.selectedProviderId = '';
+ }
+ }
}
diff --git
a/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.html
b/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.html
index 399b5c5036..b190cdd024 100644
---
a/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.html
+++
b/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.html
@@ -52,9 +52,19 @@
color="accent"
data-cy="download-configuration-download-btn"
(click)="setCleanUp()"
+ [disabled]="requiresExportValidation() && !isExportValid()"
>
{{ 'Start Sync' | translate }}
</button>
+
+ <button
+ mat-flat-button
+ color="accent"
+ data-cy="download-configuration-download-btn"
+ (click)="deleteCleanUp()"
+ >
+ {{ 'Delete Sync' | translate }}
+ </button>
<button
mat-flat-button
class="mat-basic"
diff --git
a/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.ts
b/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.ts
index f1f5139fd7..f7d7c90d84 100644
---
a/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.ts
+++
b/ui/src/app/configuration/dialog/data-retention-dialog/data-retention-dialog.component.ts
@@ -16,13 +16,15 @@
*
*/
-import { Component, Input, OnInit, ViewChild } from '@angular/core';
+import { Component, inject, Input, OnInit, ViewChild } from '@angular/core';
import { DialogRef } from '@streampipes/shared-ui';
import { DataRetentionDialogModel } from './model/data-retention-dialog.model';
import {
DatalakeRestService,
+ ExportProviderSettings,
RetentionTimeConfig,
} from '@streampipes/platform-services';
+import { ExportProviderService } from
'projects/streampipes/platform-services/src/lib/apis/export-provider.service';
@Component({
selector: 'sp-data-retention-dialog',
@@ -34,14 +36,13 @@ export class DataRetentionDialogComponent implements OnInit
{
@Input() dataRetentionDialogModel: DataRetentionDialogModel;
retentionConfig: RetentionTimeConfig;
+ exportProvider: ExportProviderSettings;
@Input()
measurementIndex: string;
- constructor(
- public dialogRef: DialogRef<DataRetentionDialogComponent>,
- private datalakeRestService: DatalakeRestService,
- ) {}
+ dialogRef = inject(DialogRef<DataRetentionDialogComponent>);
+ datalakeRestService = inject(DatalakeRestService);
ngOnInit() {
this.datalakeRestService
@@ -60,17 +61,14 @@ export class DataRetentionDialogComponent implements OnInit
{
interval: 'DAILY',
action: 'DELETE',
},
- exportConfig: {
+ retentionExportConfig: {
exportConfig: {
format: 'csv',
csvDelimiter: 'comma',
missingValueBehaviour: 'ignore',
headerColumnName: 'key',
},
- exportProviderSettings: {
- providerType: 'local',
- path: './output',
- },
+ exportProviderId: '',
},
} as RetentionTimeConfig);
}
@@ -96,4 +94,40 @@ export class DataRetentionDialogComponent implements OnInit {
this.close(true);
});
}
+
+ deleteCleanUp() {
+ this.datalakeRestService
+ .deleteCleanup(this.measurementIndex)
+ .subscribe(data => {
+ this.close(true);
+ });
+ }
+
+ requiresExportValidation(): boolean {
+ const action = this.retentionConfig?.dataRetentionConfig?.action;
+ return action === 'SAVE' || action === 'SAVEDELETE';
+ }
+
+ isExportValid(): boolean {
+ const exportConfig =
+ this.retentionConfig?.retentionExportConfig?.exportConfig;
+ const providerId =
+ this.retentionConfig?.retentionExportConfig?.exportProviderId;
+ if (!exportConfig?.format) {
+ console.error('Export format is required.');
+ return false;
+ }
+
+ if (exportConfig.format === 'csv' && !exportConfig.csvDelimiter) {
+ console.error('CSV delimiter is required for CSV format.');
+ return false;
+ }
+
+ if (providerId == '') {
+ console.error('S3 provider details must be selected.');
+ return false;
+ }
+
+ return true;
+ }
}
diff --git
a/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.html
b/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.html
new file mode 100644
index 0000000000..6c2cbee36c
--- /dev/null
+++
b/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.html
@@ -0,0 +1,82 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div class="sp-dialog-container">
+ <div class="sp-dialog-content p-15">
+ <div *ngIf="!isInProgress" fxLayout="column">
+ <div fxFlex="100" fxLayoutAlign="center center" fxLayout="column">
+ <b
+ ><h4>
+ {{
+ 'Do you really want to delete the export provider?'
+ | translate
+ }}
+ </h4></b
+ >
+ <b
+ ><h5>
+ {{
+ 'This operation cannot be undone. Please ensure
that the data provider is not used in a datalake retention.'
+ | translate
+ }}
+ </h5></b
+ >
+ </div>
+
+ <div fxFlex="100" fxLayoutAlign="center center" fxLayout="column">
+ <button
+ mat-button
+ mat-flat-button
+ color="accent"
+ data-cy="confirm-delete-data-btn"
+ (click)="deleteExportProvider()"
+ >
+ {{ 'Delete Data' | translate }}
+ </button>
+ </div>
+ </div>
+ <div
+ fxFlex="100"
+ fxLayoutAlign="center center"
+ fxLayout="column"
+ *ngIf="isInProgress"
+ data-cy="adapter-deletion-in-progress"
+ >
+ <div fxLayout="row" fxLayoutAlign="space-around">
+ <mat-spinner
+ [mode]="'indeterminate'"
+ color="accent"
+ ></mat-spinner>
+ </div>
+ <b
+ ><h4>{{ currentStatus }}</h4></b
+ >
+ </div>
+ </div>
+ <mat-divider></mat-divider>
+ <div class="sp-dialog-actions actions-align-right">
+ <button
+ mat-button
+ mat-flat-button
+ class="mat-basic"
+ (click)="close(false)"
+ >
+ Close
+ </button>
+ </div>
+</div>
diff --git
a/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.ts
b/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.ts
new file mode 100644
index 0000000000..9349fcd2aa
--- /dev/null
+++
b/ui/src/app/configuration/dialog/delete-export-provider/delete-export-provider-dialog.component.ts
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, inject, Input } from '@angular/core';
+import { DialogRef } from '@streampipes/shared-ui';
+import { ExportProviderService } from
'projects/streampipes/platform-services/src/lib/apis/export-provider.service';
+
+@Component({
+ selector: 'sp-delete-export-provider-dialog',
+ templateUrl: './delete-export-provider-dialog.component.html',
+ standalone: false,
+})
+export class DeleteExportProviderComponent {
+ @Input()
+ providerId: string;
+
+ private dialogRef = inject(DialogRef<DeleteExportProviderComponent>);
+ private exportProviderRestService = inject(ExportProviderService);
+
+ isInProgress = false;
+ currentStatus: any;
+
+ close(refreshDataLakeIndex: boolean) {
+ this.dialogRef.close(refreshDataLakeIndex);
+ }
+
+ deleteExportProvider() {
+ this.isInProgress = true;
+ this.currentStatus = 'Deleting export provider.';
+ this.exportProviderRestService
+ .deleteExportProvider(this.providerId)
+ .subscribe(data => {
+ this.close(true);
+ });
+ }
+}
diff --git
a/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.html
b/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.html
new file mode 100644
index 0000000000..883197ccd0
--- /dev/null
+++
b/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.html
@@ -0,0 +1,121 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+<div class="sp-dialog-container">
+ <div class="sp-dialog-content p-15">
+ <form [formGroup]="exportForm">
+ <mat-form-field class="sp-select-field">
+ <mat-label>{{ 'Select Provider' | translate }}</mat-label>
+ <mat-select
+ formControlName="providerType"
+ data-cy="exportProvider-select"
+ >
+ <mat-option value="S3" data-cy="exportProvider-s3">
+ {{ 'S3' | translate }}
+ </mat-option>
+ </mat-select>
+ </mat-form-field>
+ @if (exportForm.get('providerType')?.value === 'S3') {
+ <div class="s3-config-container">
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Access Key' | translate }}</mat-label>
+ <input
+ matInput
+ formControlName="accessKey"
+ placeholder="{{ 'Enter Access Key' | translate }}"
+ data-cy="exportProvider-s3-accessKey"
+ />
+ </mat-form-field>
+
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Secret Key' | translate }}</mat-label>
+ <input
+ matInput
+ type="password"
+ formControlName="secretKey"
+ placeholder="{{ 'Enter Secret Key' | translate }}"
+ data-cy="exportProvider-s3-secretKey"
+ />
+ </mat-form-field>
+
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Endpoint' | translate }}</mat-label>
+ <input
+ matInput
+ formControlName="endPoint"
+ placeholder="{{ 'Enter Endpoint' | translate }}"
+ data-cy="exportProvider-s3-endpoint"
+ />
+ @if (
+ exportForm.get('endPoint')?.hasError('invalidUri')
+ ) {
+ <mat-error
+ >{{ 'Invalid URI format.' | translate }}
+ </mat-error>
+ }
+ </mat-form-field>
+
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Bucket' | translate }}</mat-label>
+ <input
+ matInput
+ formControlName="bucketName"
+ placeholder="{{ 'Enter Bucket Name' | translate }}"
+ data-cy="exportProvider-s3-bucket"
+ />
+ </mat-form-field>
+
+ <mat-form-field class="w-100">
+ <mat-label>{{ 'Region' | translate }}</mat-label>
+ <input
+ matInput
+ formControlName="awsRegion"
+ placeholder="{{ 'Enter Region Name' | translate }}"
+ data-cy="exportProvider-s3-region"
+ />
+ </mat-form-field>
+ </div>
+ }
+
+ <mat-divider></mat-divider>
+
+ <div class="sp-dialog-actions actions-align-right">
+ <button
+ mat-button
+ mat-flat-button
+ class="mat-basic"
+ type="button"
+ (click)="addData()"
+ [disabled]="exportForm.invalid"
+ >
+ Save
+ </button>
+
+ <button
+ mat-button
+ mat-flat-button
+ class="mat-basic"
+ style="margin-left: 12px"
+ type="button"
+ (click)="close(false)"
+ >
+ Close
+ </button>
+ </div>
+ </form>
+ </div>
+</div>
diff --git
a/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.ts
b/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.ts
new file mode 100644
index 0000000000..ebf0afb436
--- /dev/null
+++
b/ui/src/app/configuration/dialog/export-provider-dialog/export-provider-dialog.component.ts
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+import { Component, inject, Input, OnInit } from '@angular/core';
+import {
+ AbstractControl,
+ FormBuilder,
+ FormGroup,
+ ValidationErrors,
+ Validators,
+} from '@angular/forms';
+import { DialogRef } from '@streampipes/shared-ui';
+import { ExportProviderSettings } from '@streampipes/platform-services';
+import { ExportProviderService } from
'projects/streampipes/platform-services/src/lib/apis/export-provider.service';
+
+@Component({
+ selector: 'sp-export-provider-dialog',
+ templateUrl: './export-provider-dialog.component.html',
+ standalone: false,
+})
+export class ExportProviderComponent implements OnInit {
+ @Input()
+ provider: ExportProviderSettings;
+
+ private dialogRef = inject<DialogRef<ExportProviderComponent>>(DialogRef);
+ private exportProviderRestService = inject(ExportProviderService);
+ private fb = inject(FormBuilder);
+
+ exportForm: FormGroup;
+
+ ngOnInit() {
+ this.initForm();
+
+ if (this.provider) {
+ this.exportForm.patchValue(this.provider);
+ }
+
+ this.exportForm.get('providerType')?.valueChanges.subscribe(type => {
+ this.toggleS3Fields(type === 'S3');
+ });
+ this.toggleS3Fields(
+ this.exportForm.get('providerType')?.value === 'S3',
+ );
+ }
+
+ initForm() {
+ this.exportForm = this.fb.group({
+ providerType: ['FOLDER', Validators.required],
+ accessKey: ['', Validators.required],
+ secretKey: ['', Validators.required],
+ endPoint: ['', [Validators.required, this.uriValidator]],
+ bucketName: ['', Validators.required],
+ awsRegion: ['US_EAST_1', Validators.required],
+ providerId: [''],
+ secretEncrypted: [false],
+ });
+ }
+
+ toggleS3Fields(enabled: boolean) {
+ const fields = [
+ 'accessKey',
+ 'secretKey',
+ 'endPoint',
+ 'bucketName',
+ 'awsRegion',
+ ];
+ fields.forEach(field => {
+ const control = this.exportForm.get(field);
+ if (enabled) {
+ control?.setValidators(Validators.required);
+ if (field === 'endPoint') {
+ control?.addValidators(this.uriValidator.bind(this));
+ }
+ control?.enable();
+ } else {
+ control?.clearValidators();
+ control?.disable();
+ }
+ control?.updateValueAndValidity();
+ });
+ }
+ uriValidator(control: AbstractControl): ValidationErrors | null {
+ const value = control.value;
+ if (!value) return null;
+
+ try {
+ new URL(value);
+ return null;
+ } catch (e) {
+ return { invalidUri: true };
+ }
+ }
+
+ addData() {
+ if (this.exportForm.invalid) {
+ this.exportForm.markAllAsTouched();
+ return;
+ }
+
+ const formValue: ExportProviderSettings = this.exportForm.value;
+
+ if (!formValue.providerId) {
+ formValue.providerId = this.makeProviderId();
+ }
+ formValue.awsRegion = formValue.awsRegion;
+
+ this.exportProviderRestService
+ .updateExportProvider(formValue)
+ .subscribe(() => this.dialogRef.close(true));
+ }
+
+ close(refresh: boolean) {
+ this.dialogRef.close(refresh);
+ }
+
+ private makeProviderId(): string {
+ return 'p' + Math.random().toString(36).substring(2, 9);
+ }
+}
