This is an automated email from the ASF dual-hosted git repository.
SvenO3 pushed a commit to branch
4633-move-dataset-count-display-into-dataset-overview
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/4633-move-dataset-count-display-into-dataset-overview by this push:
new a0b5de13b5 Add last event column to dataset view
a0b5de13b5 is described below
commit a0b5de13b5b1da969a824c91d88e84d2b47b25ec
Author: Sven Oehler <[email protected]>
AuthorDate: Thu Jun 25 10:41:30 2026 +0200
Add last event column to dataset view
---
.../last-updated-formatter.service.ts | 101 +++++++++++++++++++
.../components/kiosk/dashboard-kiosk.component.ts | 54 ++---------
.../datalake-configuration-entry.ts | 6 +-
.../datalake-configuration.component.html | 88 +----------------
.../datalake-configuration.component.ts | 107 ++++++++++++++-------
5 files changed, 184 insertions(+), 172 deletions(-)
diff --git
a/ui/src/app/core-services/time-formatting/last-updated-formatter.service.ts
b/ui/src/app/core-services/time-formatting/last-updated-formatter.service.ts
new file mode 100644
index 0000000000..4f906933b7
--- /dev/null
+++ b/ui/src/app/core-services/time-formatting/last-updated-formatter.service.ts
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { inject, Injectable } from '@angular/core';
+import { TranslateService } from '@ngx-translate/core';
+
+@Injectable({ providedIn: 'root' })
+export class LastUpdatedFormatterService {
+ private translateService = inject(TranslateService);
+
+ private exactTimeFormatter: Intl.DateTimeFormat;
+ private relativeTimeFormatter: Intl.RelativeTimeFormat;
+ private formatterLocale: string;
+
+ public formatLastUpdatedAt(
+ updatedAt: number | null | undefined,
+ currentTime = Date.now(),
+ emptyLabel = 'n/a',
+ ): string {
+ if (!updatedAt) {
+ return emptyLabel;
+ }
+
+ const exactTime = this.formatExactTime(updatedAt);
+ const relativeTime = this.formatRelativeTime(updatedAt, currentTime);
+ return relativeTime ? `${relativeTime} (${exactTime})` : exactTime;
+ }
+
+ private formatExactTime(updatedAt: number): string {
+ this.updateFormatters();
+ return this.exactTimeFormatter.format(new Date(updatedAt));
+ }
+
+ private formatRelativeTime(
+ updatedAt: number,
+ currentTime: number,
+ ): string | undefined {
+ const ageInSeconds = Math.max(
+ 0,
+ Math.round((currentTime - updatedAt) / 1000),
+ );
+
+ if (ageInSeconds < 60) {
+ this.updateFormatters();
+ return this.relativeTimeFormatter.format(-ageInSeconds, 'second');
+ } else if (ageInSeconds < 3600) {
+ this.updateFormatters();
+ return this.relativeTimeFormatter.format(
+ -Math.floor(ageInSeconds / 60),
+ 'minute',
+ );
+ } else if (ageInSeconds < 86400) {
+ this.updateFormatters();
+ return this.relativeTimeFormatter.format(
+ -Math.floor(ageInSeconds / 3600),
+ 'hour',
+ );
+ }
+
+ return undefined;
+ }
+
+ private updateFormatters(): void {
+ const locale = this.currentLocale;
+ if (this.formatterLocale === locale) {
+ return;
+ }
+
+ this.exactTimeFormatter = new Intl.DateTimeFormat(locale, {
+ dateStyle: 'medium',
+ timeStyle: 'medium',
+ });
+ this.relativeTimeFormatter = new Intl.RelativeTimeFormat(locale, {
+ numeric: 'auto',
+ });
+ this.formatterLocale = locale;
+ }
+
+ private get currentLocale(): string {
+ return (
+ this.translateService.getCurrentLang() ||
+ this.translateService.getFallbackLang() ||
+ 'en'
+ );
+ }
+}
diff --git
a/ui/src/app/dashboard-kiosk/components/kiosk/dashboard-kiosk.component.ts
b/ui/src/app/dashboard-kiosk/components/kiosk/dashboard-kiosk.component.ts
index 8686067d34..e8b62b8e3c 100644
--- a/ui/src/app/dashboard-kiosk/components/kiosk/dashboard-kiosk.component.ts
+++ b/ui/src/app/dashboard-kiosk/components/kiosk/dashboard-kiosk.component.ts
@@ -45,7 +45,8 @@ import {
LayoutAlignDirective,
LayoutDirective,
} from '@ngbracket/ngx-layout/flex';
-import { TranslatePipe, TranslateService } from '@ngx-translate/core';
+import { TranslatePipe } from '@ngx-translate/core';
+import { LastUpdatedFormatterService } from
'../../../core-services/time-formatting/last-updated-formatter.service';
@Component({
selector: 'sp-dashboard-kiosk',
@@ -66,9 +67,9 @@ export class DashboardKioskComponent implements OnInit,
OnDestroy {
private destroyRef = inject(DestroyRef);
private dashboardService = inject(DashboardService);
private timeSelectionService = inject(TimeSelectionService);
- private translateService = inject(TranslateService);
private dataExplorerDashboardService =
inject(DataExplorerDashboardService);
private dataExplorerSharedService = inject(ChartSharedService);
+ private lastUpdatedFormatterService = inject(LastUpdatedFormatterService);
observableGenerator: ObservableGenerator;
dashboard: Dashboard;
@@ -188,55 +189,12 @@ export class DashboardKioskComponent implements OnInit,
OnDestroy {
}
formatLastUpdatedAt(): string {
- const exactTime = this.formatExactLastUpdatedAt();
- const relativeTime = this.formatRelativeLastUpdatedAt();
- return relativeTime ? `${relativeTime} (${exactTime})` : exactTime;
- }
-
- private formatExactLastUpdatedAt(): string {
- return new Intl.DateTimeFormat(this.currentLocale, {
- dateStyle: 'medium',
- timeStyle: 'medium',
- }).format(new Date(this.lastUpdatedAt));
- }
-
- private formatRelativeLastUpdatedAt(): string | undefined {
- const ageInSeconds = Math.max(
- 0,
- Math.round((this.currentTime - this.lastUpdatedAt) / 1000),
- );
-
- if (ageInSeconds < 60) {
- return this.relativeTimeFormatter.format(-ageInSeconds, 'second');
- } else if (ageInSeconds < 3600) {
- return this.relativeTimeFormatter.format(
- -Math.floor(ageInSeconds / 60),
- 'minute',
- );
- } else if (ageInSeconds < 86400) {
- return this.relativeTimeFormatter.format(
- -Math.floor(ageInSeconds / 3600),
- 'hour',
- );
- }
-
- return undefined;
- }
-
- private get currentLocale(): string {
- return (
- this.translateService.currentLang ||
- this.translateService.defaultLang ||
- 'en'
+ return this.lastUpdatedFormatterService.formatLastUpdatedAt(
+ this.lastUpdatedAt,
+ this.currentTime,
);
}
- private get relativeTimeFormatter(): Intl.RelativeTimeFormat {
- return new Intl.RelativeTimeFormat(this.currentLocale, {
- numeric: 'auto',
- });
- }
-
ngOnDestroy() {
this.refresh$?.unsubscribe();
this.dashboardRefresh$?.unsubscribe();
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
index a6b3aadef5..c6f9a7f1e9 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration-entry.ts
@@ -20,10 +20,8 @@ export class DataLakeConfigurationEntry {
public name: string;
public measureName: string;
public pipelines: string[] = [];
- public eventsTotal = 0;
- public eventsLatest = 0;
- public eventsTotalLoading = false;
- public eventsLatestLoading = false;
+ public lastEvent: number | null = null;
+ public lastEventLoading = false;
public remove = true;
public elementId: string;
public retentionConfigured = false;
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
index 3b3e882896..0c00c5679f 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
@@ -49,14 +49,6 @@
</button>
</div>
<div fxLayout="column" class="w-100" fxLayoutGap="10px">
- <sp-alert-banner
- type="info"
- [title]="'Caution when loading total count' | translate"
- [description]="
- 'For large datasets, computing the total number of events
can take a long time.'
- "
- >
- </sp-alert-banner>
<sp-table
featureCardId="measurement"
[dataSource]="dataSource"
@@ -90,16 +82,16 @@
</td>
</ng-container>
- <ng-container matColumnDef="eventsLatest">
+ <ng-container matColumnDef="lastEvent">
<th mat-header-cell mat-sort-header *matHeaderCellDef>
- {{ '# Events (7d)' | translate }}
+ {{ 'Last event' | translate }}
</th>
<td
mat-cell
- data-cy="datalake-number-of-events-latest"
+ data-cy="datalake-last-event"
*matCellDef="let configurationEntry"
>
- @if (configurationEntry.eventsLatestLoading) {
+ @if (configurationEntry.lastEventLoading) {
<mat-spinner
[diameter]="20"
fxLayoutAlign="center"
@@ -107,81 +99,11 @@
>{{ 'Loading' | translate }}
</mat-spinner>
} @else {
- <sp-label
- tone="neutral"
- minWidth="100px"
- [labelText]="
- configurationEntry.eventsLatest | number
- "
- size="small"
- >
- </sp-label>
+ {{ formatLastEvent(configurationEntry.lastEvent) }}
}
</td>
</ng-container>
- <ng-container matColumnDef="eventsTotal">
- <th mat-header-cell mat-sort-header *matHeaderCellDef>
- {{ '# Events (total)' | translate }}
- </th>
- <td
- mat-cell
- data-cy="datalake-number-of-events"
- *matCellDef="let configurationEntry"
- >
- <div fxLayout="row" fxLayoutAlign="start center">
- @if (configurationEntry.eventsTotalLoading) {
- <mat-spinner
- [diameter]="20"
- fxLayoutAlign="center"
- color="accent"
- data-cy="datalake-number-of-events-spinner"
- >{{ 'Loading' | translate }}
- </mat-spinner>
- } @else {
- @if (configurationEntry.eventsTotal > -1) {
- <sp-label
- tone="neutral"
- class="cursor-pointer"
- data-cy="datalake-total-count-value"
- minWidth="100px"
- [labelText]="
- configurationEntry.eventsTotal
- | number
- "
- size="small"
- (click)="
- receiveTotalMeasurementSize(
- configurationEntry
- );
- $event.stopPropagation()
- "
- >
- </sp-label>
- } @else {
- <sp-label
- tone="neutral"
- class="cursor-pointer"
- data-cy="datalake-total-count-button"
- minWidth="100px"
- [labelText]="
- 'Click to load' | translate
- "
- size="small"
- (click)="
- receiveTotalMeasurementSize(
- configurationEntry
- );
- $event.stopPropagation()
- "
- >
- </sp-label>
- }
- }
- </div>
- </td>
- </ng-container>
-
<ng-container matColumnDef="retention">
<th mat-header-cell *matHeaderCellDef>
{{ 'Retention' | translate }}
diff --git
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
index b98ac2278a..c8977701f6 100644
---
a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
+++
b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
@@ -45,6 +45,7 @@ import {
DatasetSummaryDto,
ExportProviderService,
ExportProviderSettings,
+ SpQueryResult,
} from '@streampipes/platform-services';
import { MatPaginator } from '@angular/material/paginator';
import { MatSort, MatSortHeader } from '@angular/material/sort';
@@ -57,11 +58,9 @@ import {
ObjectPermissionDialogComponent,
PanelType,
SpAssetBrowserService,
- SpAlertBannerComponent,
SpBasicHeaderTitleComponent,
SpBasicViewComponent,
SpBreadcrumbService,
- SpLabelComponent,
SpTableAssetContextConfig,
SpTableActionsDirective,
SpTableComponent,
@@ -88,10 +87,11 @@ import { MatButton, MatIconButton } from
'@angular/material/button';
import { MatTooltip } from '@angular/material/tooltip';
import { MatIcon } from '@angular/material/icon';
import { MatProgressSpinner } from '@angular/material/progress-spinner';
-import { DatePipe, DecimalPipe, NgStyle } from '@angular/common';
+import { DatePipe, NgStyle } from '@angular/common';
import { StyleDirective } from '@ngbracket/ngx-layout/extended';
import { MatMenuItem } from '@angular/material/menu';
-import { forkJoin, Subscription } from 'rxjs';
+import { catchError, forkJoin, map, of, Subscription } from 'rxjs';
+import { LastUpdatedFormatterService } from
'../../../core-services/time-formatting/last-updated-formatter.service';
@Component({
selector: 'sp-datalake-configuration',
@@ -123,14 +123,11 @@ import { forkJoin, Subscription } from 'rxjs';
MatHeaderRow,
MatRowDef,
MatRow,
- DecimalPipe,
DatePipe,
TranslatePipe,
- SpLabelComponent,
SpTableComponent,
SpBasicHeaderTitleComponent,
SpBasicViewComponent,
- SpAlertBannerComponent,
SpTableActionsDirective,
],
})
@@ -150,6 +147,7 @@ export class DatalakeConfigurationComponent
private currentUserService = inject(CurrentUserService);
private assetFilterService = inject(SpAssetBrowserService);
private router = inject(Router);
+ private lastUpdatedFormatterService = inject(LastUpdatedFormatterService);
dataSource: MatTableDataSource<DataLakeConfigurationEntry> =
new MatTableDataSource([]);
@@ -168,8 +166,7 @@ export class DatalakeConfigurationComponent
'name',
'assetContext',
'pipeline',
- 'eventsLatest',
- 'eventsTotal',
+ 'lastEvent',
'retention',
'actions',
];
@@ -190,6 +187,7 @@ export class DatalakeConfigurationComponent
isAdmin = false;
writeAccess = false;
assetFilter$: Subscription;
+ currentTime = Date.now();
currentFilterIds: Set<string> = new Set<string>();
ngOnInit(): void {
@@ -219,7 +217,7 @@ export class DatalakeConfigurationComponent
this.spTable.paginator.page.subscribe(event => {
this.pageIndex = event.pageIndex;
this.pageSize = event.pageSize;
- this.receiveMeasurementSizes(this.pageIndex);
+ this.receiveLastEventTimes(this.pageIndex);
});
}
@@ -267,7 +265,7 @@ export class DatalakeConfigurationComponent
this.dataSource.data = this.filteredMeasurements;
this.updatePaginatorAfterFiltering();
- this.receiveMeasurementSizes(this.pageIndex);
+ this.receiveLastEventTimes(this.pageIndex);
setTimeout(() => {
this.dataSource.paginator = this.paginator;
@@ -432,21 +430,16 @@ export class DatalakeConfigurationComponent
onPageChange(event: any): void {
this.pageIndex = event.pageIndex;
this.pageSize = event.pageSize;
- //this.receiveMeasurementSizes(this.pageIndex);
}
- receiveTotalMeasurementSize(entry: DataLakeConfigurationEntry): void {
- this.queryEntryCounts([entry], 'eventsTotal');
- }
-
- receiveMeasurementSizes(pageIndex: number): void {
+ receiveLastEventTimes(pageIndex: number): void {
const start = pageIndex * this.pageSize;
const end = start + this.pageSize;
const measurements = this.filteredMeasurements
.slice(start, end)
- .filter(m => m.eventsLatest === -1);
+ .filter(m => m.lastEvent === null);
if (measurements.length > 0) {
- this.queryEntryCounts(measurements, 'eventsLatest', 7);
+ this.queryLastEventTimes(measurements);
}
}
showPermissionsDialog(element: DataLakeConfigurationEntry): void {
@@ -487,38 +480,51 @@ export class DatalakeConfigurationComponent
});
}
- queryEntryCounts(
- measurements: DataLakeConfigurationEntry[],
- targetField: string,
- daysBack = -1,
- ): void {
- this.applyLoadingStatus(measurements, targetField, true);
+ queryLastEventTimes(measurements: DataLakeConfigurationEntry[]): void {
+ this.applyLastEventLoadingStatus(measurements, true);
forkJoin(
measurements.map(measurement =>
- this.datalakeRestService.getMeasurementEntryCount(
- measurement.elementId,
- daysBack,
- ),
+ this.datalakeRestService
+ .getData(
+ measurement.name,
+ {
+ endDate: Date.now(),
+ startDate: 0,
+ limit: 1,
+ order: 'DESC',
+ missingValueBehaviour: 'empty',
+ },
+ true,
+ )
+ .pipe(
+ map(result => this.extractLastTimestamp(result)),
+ catchError(() => of(0)),
+ ),
),
).subscribe(res => {
- this.applyLoadingStatus(measurements, targetField, false);
+ this.applyLastEventLoadingStatus(measurements, false);
measurements.forEach((measurement, index) => {
- measurement[targetField] = res[index];
+ measurement.lastEvent = res[index];
});
});
}
- applyLoadingStatus(
+ applyLastEventLoadingStatus(
measurements: DataLakeConfigurationEntry[],
- targetField: string,
status: boolean,
): void {
- const loadingField = targetField + 'Loading';
measurements.forEach(measurement => {
- measurement[loadingField] = status;
+ measurement.lastEventLoading = status;
});
}
+ formatLastEvent(lastEventAt: number | null): string {
+ return this.lastUpdatedFormatterService.formatLastUpdatedAt(
+ lastEventAt,
+ this.currentTime,
+ );
+ }
+
private toConfigurationEntry(
measurement: DatasetSummaryDto,
): DataLakeConfigurationEntry {
@@ -531,11 +537,38 @@ export class DatalakeConfigurationComponent
entry.lastExport = measurement.lastExport;
entry.lastRetentionStatus = measurement.lastRetentionStatus;
entry.remove = measurement.removable;
- entry.eventsLatest = -1;
- entry.eventsTotal = -1;
+ entry.lastEvent = null;
return entry;
}
+ private extractLastTimestamp(result: SpQueryResult): number {
+ if (result?.lastTimestamp) {
+ return result.lastTimestamp;
+ }
+
+ const series = result?.allDataSeries?.find(
+ dataSeries => dataSeries.rows?.length > 0,
+ );
+ const row = series?.rows?.[0];
+ if (!row) {
+ return 0;
+ }
+
+ const headers = result.headers?.length
+ ? result.headers
+ : series.headers;
+ const timestampIndex = headers?.indexOf('time') ?? 0;
+ return this.toTimestamp(row[timestampIndex >= 0 ? timestampIndex : 0]);
+ }
+
+ private toTimestamp(value: unknown): number {
+ const timestamp =
+ typeof value === 'number'
+ ? value
+ : new Date(String(value)).getTime();
+ return Number.isNaN(timestamp) ? 0 : timestamp;
+ }
+
private openRetentionLogDialog(measurement: DataLakeMeasure): void {
const dialogRef: DialogRef<DataRetentionLogDialogComponent> =
this.dialogService.open(DataRetentionLogDialogComponent, {
