This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 8fd62bacc1 feat(#4072): Reduce loading time for dataset counts (#4073)
8fd62bacc1 is described below
commit 8fd62bacc1d22934625a842118dc3d8210a626df
Author: Dominik Riemer <[email protected]>
AuthorDate: Tue Dec 30 19:58:22 2025 +0100
feat(#4072): Reduce loading time for dataset counts (#4073)
---
.../dataexplorer/api/IDataExplorerManager.java | 3 +-
.../influx/DataExplorerManagerInflux.java | 5 +-
.../influx/DataLakeMeasurementCounterInflux.java | 14 +-
.../iotdb/DataExplorerManagerIotDb.java | 6 +-
.../iotdb/DataLakeMeasurementCounterIotDb.java | 9 +-
.../query/DataLakeMeasurementCounter.java | 5 +-
.../query/DataLakeMeasurementCounterTestImpl.java | 4 +-
.../impl/datalake/DataLakeMeasureResource.java | 6 +-
.../utils/dataExplorer/DataExplorerUtils.ts | 1 +
.../tests/dataExplorer/configuration.smoke.spec.ts | 4 +
ui/deployment/feature-cards.yml | 4 +
.../src/lib/apis/datalake-rest.service.ts | 6 +-
.../datalake-configuration-entry.ts | 5 +-
.../datalake-configuration.component.html | 253 ++++++++++++---------
.../datalake-configuration.component.scss | 4 +
.../datalake-configuration.component.ts | 61 +++--
.../dataset-feature-card.component.html | 90 ++++++++
.../dataset-feature-card.component.scss} | 19 +-
.../dataset-feature-card.component.ts | 108 +++++++++
ui/src/app/dataset/dataset.module.ts | 1 +
.../matching-error/matching-error.component.html | 69 +++---
21 files changed, 492 insertions(+), 185 deletions(-)
diff --git
a/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerManager.java
b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerManager.java
index 586a77706c..5711aade33 100644
---
a/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerManager.java
+++
b/streampipes-data-explorer-api/src/main/java/org/apache/streampipes/dataexplorer/api/IDataExplorerManager.java
@@ -35,7 +35,8 @@ public interface IDataExplorerManager {
*/
IDataLakeMeasurementCounter getMeasurementCounter(
List<DataLakeMeasure> allMeasurements,
- List<String> measurementsToCount
+ List<String> measurementsToCount,
+ int daysBack
);
IDataExplorerQueryManagement
getQueryManagement(IDataExplorerSchemaManagement dataExplorerSchemaManagement);
diff --git
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerManagerInflux.java
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerManagerInflux.java
index 869f314b1d..f88cf6c585 100644
---
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerManagerInflux.java
+++
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataExplorerManagerInflux.java
@@ -39,8 +39,9 @@ public class DataExplorerManagerInflux implements
IDataExplorerManager {
@Override
public IDataLakeMeasurementCounter getMeasurementCounter(
List<DataLakeMeasure> allMeasurements,
- List<String> measurementsToCount) {
- return new DataLakeMeasurementCounterInflux(allMeasurements,
measurementsToCount);
+ List<String> measurementsToCount,
+ int daysBack) {
+ return new DataLakeMeasurementCounterInflux(allMeasurements,
measurementsToCount, daysBack);
}
@Override
diff --git
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCounterInflux.java
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCounterInflux.java
index 0a58b8b29a..a54a3db9e0 100644
---
a/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCounterInflux.java
+++
b/streampipes-data-explorer-influx/src/main/java/org/apache/streampipes/dataexplorer/influx/DataLakeMeasurementCounterInflux.java
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
public class DataLakeMeasurementCounterInflux extends
DataLakeMeasurementCounter {
@@ -37,9 +38,10 @@ public class DataLakeMeasurementCounterInflux extends
DataLakeMeasurementCounter
public DataLakeMeasurementCounterInflux(
List<DataLakeMeasure> allMeasurements,
- List<String> measurementNames
+ List<String> measurementNames,
+ int daysBack
) {
- super(allMeasurements, measurementNames);
+ super(allMeasurements, measurementNames, daysBack);
}
@Override
@@ -54,10 +56,16 @@ public class DataLakeMeasurementCounterInflux extends
DataLakeMeasurementCounter
return 0;
}
+ var endTime = System.currentTimeMillis();
+ long startTime = endTime - TimeUnit.DAYS.toMillis(daysBack);
var builder = DataLakeInfluxQueryBuilder
.create(measure.getMeasureName())
- .withEndTime(System.currentTimeMillis())
+ .withEndTime(endTime)
.withAggregatedColumn(firstColumn, AggregationFunction.COUNT);
+
+ if (daysBack > -1) {
+ builder.withStartTime(startTime);
+ }
var queryResult = new
DataExplorerInfluxQueryExecutor().executeQuery(builder.build(),
Optional.empty(), true);
return queryResult.getTotal() > 0 ? extractResult(queryResult,
COUNT_FIELD) : 0;
diff --git
a/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataExplorerManagerIotDb.java
b/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataExplorerManagerIotDb.java
index 612c2dee6c..e931403809 100644
---
a/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataExplorerManagerIotDb.java
+++
b/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataExplorerManagerIotDb.java
@@ -36,8 +36,10 @@ import java.util.List;
public class DataExplorerManagerIotDb implements IDataExplorerManager {
@Override
- public IDataLakeMeasurementCounter
getMeasurementCounter(List<DataLakeMeasure> allMeasurements, List<String>
measurementsToCount) {
- return new DataLakeMeasurementCounterIotDb(allMeasurements,
measurementsToCount);
+ public IDataLakeMeasurementCounter
getMeasurementCounter(List<DataLakeMeasure> allMeasurements,
+ List<String>
measurementsToCount,
+ int daysBack) {
+ return new DataLakeMeasurementCounterIotDb(allMeasurements,
measurementsToCount, daysBack);
}
@Override
diff --git
a/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataLakeMeasurementCounterIotDb.java
b/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataLakeMeasurementCounterIotDb.java
index 14cbf5aaf7..83470d4b78 100644
---
a/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataLakeMeasurementCounterIotDb.java
+++
b/streampipes-data-explorer-iotdb/src/main/java/org/apache/streampipes/dataexplorer/iotdb/DataLakeMeasurementCounterIotDb.java
@@ -18,12 +18,12 @@
package org.apache.streampipes.dataexplorer.iotdb;
-import org.apache.iotdb.rpc.IoTDBConnectionException;
-import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.streampipes.commons.environment.Environments;
import org.apache.streampipes.dataexplorer.query.DataLakeMeasurementCounter;
import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.iotdb.rpc.IoTDBConnectionException;
+import org.apache.iotdb.rpc.StatementExecutionException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -35,8 +35,9 @@ public class DataLakeMeasurementCounterIotDb extends
DataLakeMeasurementCounter
private static final Logger LOG =
LoggerFactory.getLogger(DataLakeMeasurementCounterIotDb.class);
public DataLakeMeasurementCounterIotDb(List<DataLakeMeasure> allMeasurements,
- List<String> measurementNames) {
- super(allMeasurements, measurementNames);
+ List<String> measurementNames,
+ int daysBack) {
+ super(allMeasurements, measurementNames, daysBack);
}
/**
diff --git
a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounter.java
b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounter.java
index bc3a551ed7..2263c06192 100644
---
a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounter.java
+++
b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounter.java
@@ -41,13 +41,16 @@ public abstract class DataLakeMeasurementCounter implements
IDataLakeMeasurement
protected final List<DataLakeMeasure> allMeasurements;
protected final List<String> measurementNames;
+ protected final int daysBack;
public DataLakeMeasurementCounter(
List<DataLakeMeasure> allMeasurements,
- List<String> measurementNames
+ List<String> measurementNames,
+ int daysBack
) {
this.allMeasurements = allMeasurements;
this.measurementNames = measurementNames;
+ this.daysBack = daysBack;
}
@Override
diff --git
a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounterTestImpl.java
b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounterTestImpl.java
index 4642c5f227..20013aad9d 100644
---
a/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounterTestImpl.java
+++
b/streampipes-data-explorer/src/test/java/org/apache/streampipes/dataexplorer/query/DataLakeMeasurementCounterTestImpl.java
@@ -27,7 +27,7 @@ public class DataLakeMeasurementCounterTestImpl extends
DataLakeMeasurementCount
public DataLakeMeasurementCounterTestImpl(
List<DataLakeMeasure> allMeasurements, List<String> measurementNames
) {
- super(allMeasurements, measurementNames);
+ super(allMeasurements, measurementNames, -1);
}
@Override
@@ -35,4 +35,4 @@ public class DataLakeMeasurementCounterTestImpl extends
DataLakeMeasurementCount
// Mock implementation for testing
return CompletableFuture.completedFuture(1);
}
-}
\ No newline at end of file
+}
diff --git
a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeMeasureResource.java
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeMeasureResource.java
index 32d6a26293..4c36c5a06c 100644
---
a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeMeasureResource.java
+++
b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeMeasureResource.java
@@ -69,13 +69,15 @@ public class DataLakeMeasureResource extends
AbstractAuthGuardedRestResource {
@Operation(summary = "Retrieve measurement counts", description = "Retrieves
the entry counts for the specified measurements from the data lake.")
@GetMapping(path = "/count", produces = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<Map<String, Integer>> getEntryCountsOfMeasurments(
- @Parameter(description = "A list of measurement names to return the
count.") @RequestParam(value = "measurementNames") List<String>
measurementNames) {
+ @Parameter(description = "A list of measurement names to return the
count.") @RequestParam(value = "measurementNames") List<String>
measurementNames,
+ @Parameter(description = "The number of days from today where the count
should start") @RequestParam(value = "daysBack", defaultValue = "-1") int
daysBack) {
var allMeasurements = this.dataLakeMeasureManagement.getAllMeasurements();
var result = new DataExplorerDispatcher()
.getDataExplorerManager()
.getMeasurementCounter(
allMeasurements,
- measurementNames)
+ measurementNames,
+ daysBack)
.countMeasurementSizes();
return ok(result);
}
diff --git a/ui/cypress/support/utils/dataExplorer/DataExplorerUtils.ts
b/ui/cypress/support/utils/dataExplorer/DataExplorerUtils.ts
index a34bf7d20e..88194fa69a 100644
--- a/ui/cypress/support/utils/dataExplorer/DataExplorerUtils.ts
+++ b/ui/cypress/support/utils/dataExplorer/DataExplorerUtils.ts
@@ -704,6 +704,7 @@ export class DataExplorerUtils {
}
public static waitForCountingResults() {
+ cy.dataCy('datalake-total-count-button').click();
cy.dataCy('datalake-number-of-events-spinner', {
timeout: 10000,
}).should('exist');
diff --git a/ui/cypress/tests/dataExplorer/configuration.smoke.spec.ts
b/ui/cypress/tests/dataExplorer/configuration.smoke.spec.ts
index 0033da2d30..10bf20f5ff 100644
--- a/ui/cypress/tests/dataExplorer/configuration.smoke.spec.ts
+++ b/ui/cypress/tests/dataExplorer/configuration.smoke.spec.ts
@@ -30,6 +30,7 @@ describe('Test Truncate data in datalake', () => {
it('Perform Test', () => {
DataExplorerUtils.goToDatalakeConfiguration();
+ cy.dataCy('datalake-total-count-button').click();
// Check if amount of events is correct
DataExplorerBtns.datalakeNumberEvents()
@@ -43,6 +44,8 @@ describe('Test Truncate data in datalake', () => {
.should('be.visible')
.click();
+ cy.dataCy('datalake-total-count-button').click();
+
// Check if amount of events is zero. The should('have.text, '0') is
required to check for text equality
DataExplorerBtns.datalakeNumberEvents()
.should('be.visible')
@@ -62,6 +65,7 @@ describe('Delete data in datalake', () => {
it('Perform Test', () => {
DataExplorerUtils.goToDatalakeConfiguration();
+ cy.dataCy('datalake-total-count-button').click();
// Check if amount of events is correct
DataExplorerBtns.datalakeNumberEvents()
diff --git a/ui/deployment/feature-cards.yml b/ui/deployment/feature-cards.yml
index e00d829d4a..27b4dded52 100644
--- a/ui/deployment/feature-cards.yml
+++ b/ui/deployment/feature-cards.yml
@@ -25,3 +25,7 @@
componentPath:
'./dashboard/components/dashboard-feature-card/dashboard-feature-card.component'
componentName: DashboardFeatureCardComponent
moduleName: spDashboard
+- id: measurement
+ componentPath:
'./dataset/components/dataset-feature-card/dataset-feature-card.component'
+ componentName: DatasetFeatureCardComponent
+ moduleName: spDatasets
diff --git
a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index bade947937..b119cd6296 100644
---
a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++
b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -50,10 +50,14 @@ export class DatalakeRestService {
getMeasurementEntryCounts(
measurementNames: string[],
+ daysBack = -1,
): Observable<Record<string, number>> {
return this.http
.get(`${this.dataLakeMeasureUrl}/count`, {
- params: { measurementNames: measurementNames },
+ params: {
+ measurementNames,
+ daysBack,
+ },
})
.pipe(map(r => r as Record<string, number>));
}
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
index a56a29e6e5..8e814381f8 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
@@ -21,7 +21,10 @@ import { RetentionConfig } from
'../../dialog/data-retention-dialog/model/retent
export class DataLakeConfigurationEntry {
public name: string;
public pipelines: string[] = [];
- public events = 0;
+ public eventsTotal = 0;
+ public eventsLatest = 0;
+ public eventsTotalLoading = false;
+ public eventsLatestLoading = false;
public remove = true;
public elementId: string;
public retention: RetentionTimeConfig;
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
index bbcdeb81ab..a6aa18e361 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
@@ -38,8 +38,17 @@
<mat-icon>refresh</mat-icon>
</button>
</div>
- <div fxLayout="column" class="w-100">
+ <div fxLayout="column" class="w-100" fxLayoutGap="10px">
+ <sp-alert-banner
+ type="info"
+ [title]="'Caution when loading total count' | translate"
+ [description]="
+ 'For large datasets, computing the total number of events
can take a long time.'
+ "
+ >
+ </sp-alert-banner>
<sp-table
+ featureCardId="measurement"
[dataSource]="dataSource"
[columns]="displayedColumns"
[showActionsMenu]="true"
@@ -64,141 +73,165 @@
</td>
</ng-container>
- <ng-container matColumnDef="events">
+ <ng-container matColumnDef="eventsLatest">
<th mat-header-cell mat-sort-header *matHeaderCellDef>
- {{ '# Events' | translate }}
+ {{ '# Events (7d)' | translate }}
</th>
- <td
- mat-cell
- data-cy="datalake-number-of-events"
- *matCellDef="let configurationEntry"
- >
- @if (configurationEntry.events < 0) {
+ <td mat-cell *matCellDef="let configurationEntry">
+ @if (configurationEntry.eventsLatestLoading) {
<mat-spinner
[diameter]="20"
fxLayoutAlign="center"
- style="margin: 10px 0 5px 0"
color="accent"
- data-cy="datalake-number-of-events-spinner"
>{{ 'Loading' | translate }}
</mat-spinner>
} @else {
- <span>
- {{ configurationEntry.events | number }}
- </span>
+ <sp-label
+ tone="neutral"
+ minWidth="100px"
+ [labelText]="
+ configurationEntry.eventsLatest | number
+ "
+ size="small"
+ >
+ </sp-label>
}
</td>
</ng-container>
- <ng-container matColumnDef="retention">
- <th mat-header-cell *matHeaderCellDef>
- {{ 'Retention Rate' | translate }}
+ <ng-container matColumnDef="eventsTotal">
+ <th mat-header-cell mat-sort-header *matHeaderCellDef>
+ {{ '# Events (total)' | translate }}
</th>
- <td mat-cell *matCellDef="let configurationEntry">
- <div fxLayout="row">
- <span
- fxFlex
- fxFlexOrder="3"
- fxLayout="row"
- fxLayoutAlign="start center"
- >
- <button
+ <td
+ mat-cell
+ data-cy="datalake-number-of-events"
+ *matCellDef="let configurationEntry"
+ >
+ <div fxLayout="row" fxLayoutAlign="start center">
+ @if (configurationEntry.eventsTotalLoading) {
+ <mat-spinner
+ [diameter]="20"
+ fxLayoutAlign="center"
color="accent"
- mat-icon-button
- [matTooltip]="
- 'Set retention rate' | translate
- "
- data-cy="datalake-retention-btn"
- matTooltipPosition="above"
- (click)="
- openRetentionDialog(
- configurationEntry.elementId
- )
- "
- >
- <i
- class="material-icons"
- [ngStyle]="{
- color:
configurationEntry?.retention
- ? 'var(--color-success)'
- : 'var(--color-neutral',
- }"
- >history</i
+ data-cy="datalake-number-of-events-spinner"
+ >{{ 'Loading' | translate }}
+ </mat-spinner>
+ } @else {
+ @if (configurationEntry.eventsTotal > -1) {
+ <sp-label
+ tone="neutral"
+ class="cursor-pointer"
+ minWidth="100px"
+ [labelText]="
+ configurationEntry.eventsTotal
+ | number
+ "
+ size="small"
+ (click)="
+ receiveTotalMeasurementSize(
+ configurationEntry
+ )
+ "
>
- </button>
- </span>
+ </sp-label>
+ } @else {
+ <sp-label
+ tone="neutral"
+ class="cursor-pointer"
+ data-cy="datalake-total-count-button"
+ minWidth="100px"
+ [labelText]="
+ 'Click to load' | translate
+ "
+ size="small"
+ (click)="
+ receiveTotalMeasurementSize(
+ configurationEntry
+ )
+ "
+ >
+ </sp-label>
+ }
+ }
</div>
</td>
</ng-container>
- <ng-container matColumnDef="retentionlog">
+ <ng-container matColumnDef="retention">
<th mat-header-cell *matHeaderCellDef>
- {{ 'Retention Log' | translate }}
+ {{ 'Retention' | translate }}
</th>
<td mat-cell *matCellDef="let configurationEntry">
- <div fxLayout="row">
- <span
- fxFlex
- fxFlexOrder="3"
- fxLayout="row"
- fxLayoutAlign="start center"
+ <button
+ color="accent"
+ mat-icon-button
+ [matTooltip]="'Set retention rate' | translate"
+ data-cy="datalake-retention-btn"
+ matTooltipPosition="above"
+ (click)="
+ openRetentionDialog(
+ configurationEntry.elementId
+ )
+ "
+ >
+ <i
+ class="material-icons"
+ [ngStyle]="{
+ color: configurationEntry?.retention
+ ? 'var(--color-success)'
+ : 'var(--color-neutral',
+ }"
+ >history</i
+ >
+ </button>
+ @if (
+
configurationEntry?.retention?.retentionExportConfig
+ ?.retentionLog?.length > 0
+ ) {
+ <button
+ color="accent"
+ mat-icon-button
+ [matTooltip]="
+ ('Open Retention Log' | translate) +
+ (configurationEntry?.retention
+ ?.retentionExportConfig?.lastExport
+ ? ' • ' +
+ (configurationEntry.retention
+ .retentionExportConfig.lastExport
+ | date: 'yyyy-MM-dd HH:mm:ss')
+ : '')
+ "
+ data-cy="datalake-retention-log-btn"
+ matTooltipPosition="above"
+ (click)="
+ openRetentionLog(
+ configurationEntry?.retention
+ .retentionExportConfig.retentionLog
+ )
+ "
>
- @if (configurationEntry?.retention) {
- <button
- color="accent"
- mat-icon-button
- [matTooltip]="
- ('Open Retention Log' | translate)
+
- (configurationEntry?.retention
+ <i
+ class="material-icons"
+ [ngStyle]="{
+ color:
+ configurationEntry?.retention
?.retentionExportConfig
- ?.lastExport
- ? ' • ' +
- (configurationEntry.retention
- .retentionExportConfig
- .lastExport
- | date
- : 'yyyy-MM-dd
HH:mm:ss')
- : '')
- "
- data-cy="datalake-retention-log-btn"
- matTooltipPosition="above"
- (click)="
- openRetentionLog(
- configurationEntry?.retention
+ ?.retentionLog?.length &&
+ configurationEntry.retention
+ .retentionExportConfig
+ .retentionLog[
+ configurationEntry.retention
.retentionExportConfig
- .retentionLog
- )
- "
- >
- <i
- class="material-icons"
- [ngStyle]="{
- color:
- configurationEntry
- ?.retention
- ?.retentionExportConfig
- ?.retentionLog
- ?.length &&
-
configurationEntry.retention
- .retentionExportConfig
- .retentionLog[
- configurationEntry
- .retention
-
.retentionExportConfig
- .retentionLog
- .length - 1
- ].status
- ? 'green'
- : 'red',
- }"
- >list_alt</i
- >
- </button>
- } @else {
- <p>-</p>
- }
- </span>
- </div>
+ .retentionLog.length - 1
+ ].status
+ ? 'var(--color-success)'
+ : 'var(--color-error)',
+ }"
+ >list_alt</i
+ >
+ </button>
+ }
</td>
</ng-container>
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
index 13cbc4aacb..b93bc4701c 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
@@ -15,3 +15,7 @@
* limitations under the License.
*
*/
+
+.cursor-pointer {
+ cursor: pointer;
+}
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
index 6779808799..80a3d7841f 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
@@ -51,6 +51,7 @@ import { DeleteExportProviderComponent } from
'../../dialog/delete-export-provid
import { TranslateService } from '@ngx-translate/core';
import { ExportProviderConnectionTestComponent } from
'../../dialog/export-provider-connection-test/export-provider-connection-test.component';
import { DataRetentionLogDialogComponent } from
'../../dialog/data-retention-log-dialog/data-retention-log-dialog.component';
+import { LocalStorageService } from
'../../../../../projects/streampipes/shared-ui/src/lib/services/local-storage-settings.service';
@Component({
selector: 'sp-datalake-configuration',
@@ -82,9 +83,9 @@ export class DatalakeConfigurationComponent implements
OnInit, AfterViewInit {
displayedColumns: string[] = [
'name',
'pipeline',
- 'events',
+ 'eventsLatest',
+ 'eventsTotal',
'retention',
- 'retentionlog',
'actions',
];
@@ -97,7 +98,9 @@ export class DatalakeConfigurationComponent implements
OnInit, AfterViewInit {
'test',
];
- pageSize = 10;
+ private localStorageService = inject(LocalStorageService);
+
+ pageSize = this.localStorageService.get('paginator-page-size', 10);
pageIndex = 0;
ngOnInit(): void {
@@ -144,7 +147,8 @@ export class DatalakeConfigurationComponent implements
OnInit, AfterViewInit {
const entry = new DataLakeConfigurationEntry();
entry.elementId = measurement.elementId;
entry.name = measurement.measureName;
- entry.events = -1;
+ entry.eventsLatest = -1;
+ entry.eventsTotal = -1;
if (measurement?.retentionTime != null) {
entry.retention = measurement.retentionTime;
}
@@ -327,7 +331,11 @@ export class DatalakeConfigurationComponent implements
OnInit, AfterViewInit {
onPageChange(event: any) {
this.pageIndex = event.pageIndex;
this.pageSize = event.pageSize;
- this.receiveMeasurementSizes(this.pageIndex);
+ //this.receiveMeasurementSizes(this.pageIndex);
+ }
+
+ receiveTotalMeasurementSize(entry: DataLakeConfigurationEntry) {
+ this.queryEntryCounts([entry.name], 'eventsTotal');
}
receiveMeasurementSizes(pageIndex: number) {
@@ -335,18 +343,41 @@ export class DatalakeConfigurationComponent implements
OnInit, AfterViewInit {
const end = start + this.pageSize;
const measurements = this.availableMeasurements
.slice(start, end)
- .filter(m => m.events === -1)
+ .filter(m => m.eventsLatest === -1)
.map(m => m.name);
if (measurements.length > 0) {
- this.datalakeRestService
- .getMeasurementEntryCounts(measurements)
- .subscribe(res => {
- this.availableMeasurements.forEach(m => {
- if (res[m.name] !== undefined) {
- m.events = res[m.name];
- }
- });
- });
+ this.queryEntryCounts(measurements, 'eventsLatest', 7);
}
}
+
+ queryEntryCounts(
+ measurements: string[],
+ targetField: string,
+ daysBack = -1,
+ ): void {
+ this.applyLoadingStatus(measurements, targetField, true);
+ this.datalakeRestService
+ .getMeasurementEntryCounts(measurements, daysBack)
+ .subscribe(res => {
+ this.applyLoadingStatus(measurements, targetField, false);
+ this.availableMeasurements.forEach(m => {
+ if (res[m.name] !== undefined) {
+ m[targetField] = res[m.name];
+ }
+ });
+ });
+ }
+
+ applyLoadingStatus(
+ measurements: string[],
+ targetField: string,
+ status: boolean,
+ ): void {
+ const loadingField = targetField + 'Loading';
+ this.availableMeasurements.forEach(m => {
+ if (measurements.includes(m.name)) {
+ m[loadingField] = status;
+ }
+ });
+ }
}
diff --git
a/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.html
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.html
new file mode 100644
index 0000000000..d5904bcc4a
--- /dev/null
+++
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.html
@@ -0,0 +1,90 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+@if (dataset) {
+ <div fxFlexFill fxLayout="column" class="feature-card-outer">
+ <sp-feature-card-header
+ [title]="dataset.measureName"
+ [icon]="assetLinkType.linkIcon"
+ [iconColor]="assetLinkType.linkColor"
+ [detailsLink]="assetLinkType.navPaths"
+ (close)="onClose?.()"
+ (onDetailsClick)="navigateToChartView()"
+ >
+ </sp-feature-card-header>
+
+ <div class="feature-card-meta-card meta-card">
+ <sp-feature-card-meta-section [label]="'Status' | translate">
+ <div
+ class="feature-card-meta-box"
+ fxLayout="row"
+ fxLayoutAlign="start center"
+ fxLayoutGap="0.75rem"
+ >
+ <mat-icon class="feature-card-meta-icon"
+ >fiber_manual_record</mat-icon
+ >
+ <div>
+ <div class="feature-card-meta-field-label">
+ {{ 'Last event' | translate }}
+ </div>
+ <sp-label
+ class="mt-xs"
+ textCase="uppercase"
+ tone="neutral"
+ size="small"
+ variant="soft"
+ >
+ {{ (lastEventTs | date: 'short') || '-' }}
+ </sp-label>
+ </div>
+ </div>
+ </sp-feature-card-meta-section>
+
+ <div class="feature-card-meta-section__label">
+ {{ 'Preview' | translate }}
+ </div>
+ <div class="dataset-preview-inner">
+ @if (dataPreview && dataPreview.total > 0) {
+ <div fxLayout="column">
+ @for (
+ header of dataPreview.headers;
+ let i = $index;
+ track $index
+ ) {
+ <div fxLayout="row" fxFlex="100">
+ <span class="text-sm" fxFlex="70">{{
+ header
+ }}</span>
+ <span class="text-sm font-bold" fxFlex="30">
+ {{
+ dataPreview.allDataSeries[0].rows[0][i]
+ }}</span
+ >
+ </div>
+ }
+ </div>
+ } @else {
+ <span class="text-sm">{{
+ 'No data available.' | translate
+ }}</span>
+ }
+ </div>
+ </div>
+ </div>
+}
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.scss
similarity index 78%
copy from
ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
copy to
ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.scss
index 13cbc4aacb..f224c48e64 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.scss
+++
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.scss
@@ -1,4 +1,4 @@
-/*
+/*!
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -15,3 +15,20 @@
* limitations under the License.
*
*/
+
+:host {
+ width: 100%;
+ height: 100%;
+}
+
+.dataset-preview-inner {
+ overflow-y: scroll;
+ min-height: 0;
+}
+
+.meta-card {
+ display: flex;
+ flex-direction: column;
+ flex: 1 1 auto;
+ min-height: 0;
+}
diff --git
a/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.ts
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.ts
new file mode 100644
index 0000000000..fab3b673b0
--- /dev/null
+++
b/ui/src/app/dataset/components/dataset-feature-card/dataset-feature-card.component.ts
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { Component, inject, Input, OnInit } from '@angular/core';
+import {
+ DefaultFlexDirective,
+ DefaultLayoutAlignDirective,
+ DefaultLayoutDirective,
+ DefaultLayoutGapDirective,
+ FlexFillDirective,
+} from '@ngbracket/ngx-layout';
+import { SharedUiModule } from '@streampipes/shared-ui';
+import { DashboardSharedModule } from
'../../../dashboard-shared/dashboard-shared.module';
+import {
+ AssetConstants,
+ AssetLinkType,
+ DataLakeMeasure,
+ DatalakeRestService,
+ GenericStorageService,
+ SpQueryResult,
+} from '@streampipes/platform-services';
+import { forkJoin } from 'rxjs';
+import { TranslatePipe } from '@ngx-translate/core';
+import { MatIcon } from '@angular/material/icon';
+import { DatePipe } from '@angular/common';
+
+@Component({
+ selector: 'sp-dataset-feature-card',
+ templateUrl: './dataset-feature-card.component.html',
+ styleUrls: ['./dataset-feature-card.component.scss'],
+ imports: [
+ SharedUiModule,
+ FlexFillDirective,
+ DashboardSharedModule,
+ DefaultFlexDirective,
+ DefaultLayoutDirective,
+ TranslatePipe,
+ DefaultLayoutAlignDirective,
+ DefaultLayoutGapDirective,
+ MatIcon,
+ DatePipe,
+ ],
+})
+export class DatasetFeatureCardComponent implements OnInit {
+ @Input()
+ resourceId: string;
+
+ @Input()
+ onClose?: () => void;
+
+ dataset: DataLakeMeasure;
+ assetLinkType: AssetLinkType;
+ dataPreview: SpQueryResult;
+ lastEventTs: number;
+
+ private datalakeRestService = inject(DatalakeRestService);
+ private genericStorageService = inject(GenericStorageService);
+
+ ngOnInit() {
+ forkJoin([
+ this.datalakeRestService.getMeasurement(this.resourceId),
+ this.genericStorageService.getAllDocuments(
+ AssetConstants.ASSET_LINK_TYPES_DOC_NAME,
+ ),
+ ]).subscribe(res => {
+ this.dataset = res[0];
+ this.assetLinkType = res[1].find(a => a.linkType ===
'measurement');
+ this.loadSampleData();
+ });
+ }
+
+ loadSampleData(): void {
+ this.datalakeRestService
+ .getData(this.dataset.measureName, {
+ endDate: new Date().getTime(),
+ startDate: 0,
+ limit: 1,
+ order: 'DESC',
+ missingValueBehaviour: 'empty',
+ columns: this.dataset.eventSchema.eventProperties
+ .map(ep => ep.runtimeName)
+ .toString(),
+ })
+ .subscribe(res => {
+ this.dataPreview = res;
+ if (res.total > 0) {
+ this.lastEventTs = res.allDataSeries[0].rows[0][0];
+ }
+ });
+ }
+
+ navigateToChartView(): void {}
+}
diff --git a/ui/src/app/dataset/dataset.module.ts
b/ui/src/app/dataset/dataset.module.ts
index 7d64352ef6..392fe1739a 100644
--- a/ui/src/app/dataset/dataset.module.ts
+++ b/ui/src/app/dataset/dataset.module.ts
@@ -50,6 +50,7 @@ import { MatMenuModule } from '@angular/material/menu';
import { DataRetentionNowDialogComponent } from
'./dialog/data-retention-now-dialog/data-retention-now-dialog.component';
import { DataRetentionLogDialogComponent } from
'./dialog/data-retention-log-dialog/data-retention-log-dialog.component';
import { ExportProviderConnectionTestComponent } from
'./dialog/export-provider-connection-test/export-provider-connection-test.component';
+import { DatasetFeatureCardComponent } from
'./components/dataset-feature-card/dataset-feature-card.component';
@NgModule({
imports: [
diff --git
a/ui/src/app/editor/dialog/matching-error/matching-error.component.html
b/ui/src/app/editor/dialog/matching-error/matching-error.component.html
index 8d6ead9900..dcdbd0ef33 100644
--- a/ui/src/app/editor/dialog/matching-error/matching-error.component.html
+++ b/ui/src/app/editor/dialog/matching-error/matching-error.component.html
@@ -18,53 +18,42 @@
<div class="sp-dialog-container">
<div class="sp-dialog-content p-15">
- <div>
- <h4>{{ 'These elements can't be connected.' | translate }}</h4>
- <h4>
+ <div fxLayout="column">
+ <h5>{{ "These elements can't be connected." | translate }}</h5>
+ <span class="text-md">
{{
'The input data stream does not satisfy the requirements
specified by the data processor.' | translate
}}
- </h4>
- <button
- mat-button
- (click)="toggleStatusDetailsVisible()"
- type="button"
- class="md-accent"
- >
- @if (!statusDetailsVisible) {
- <div>
- {{ 'Show Details' | translate }}
- </div>
- }
- @if (statusDetailsVisible) {
- <div>
- {{ 'Hide Details' | translate }}
- </div>
- }
- </button>
- @if (statusDetailsVisible) {
- <div>
- @for (entry of notifications; track entry) {
+ </span>
+ <div>
+ <button
+ mat-flat-button
+ (click)="toggleStatusDetailsVisible()"
+ type="button"
+ class="mt-md mat-basic"
+ >
+ @if (!statusDetailsVisible) {
+ <div>
+ {{ 'Show Details' | translate }}
+ </div>
+ }
+ @if (statusDetailsVisible) {
<div>
- <div
- fxFlex="100"
- class="md-whiteframe-z1"
- style="margin-bottom: 10px"
- >
- <div
- fxFlex
- fxLayout="column"
- class="md-padding"
- >
- <div fxFlex="column">
- {{ entry.title }}<br />
- {{ entry.description }}
- </div>
- </div>
- </div>
+ {{ 'Hide Details' | translate }}
</div>
}
+ </button>
+ </div>
+ @if (statusDetailsVisible) {
+ <div class="mt-md" fxLayout="column">
+ @for (entry of notifications; track entry) {
+ <sp-alert-banner
+ type="warning"
+ [title]="entry.title"
+ [description]="entry.description"
+ ></sp-alert-banner>
+ }
</div>
}
</div>
